def main(argv): """Go Main Go""" basedir = "/mesonet/data/prism" outdir = "swatfiles_prism_arealaverage" if os.path.isdir(outdir): print("ABORT: as %s exists" % (outdir, )) return os.mkdir(outdir) for dirname in ['precipitation', 'temperature']: os.mkdir("%s/%s" % (outdir, dirname)) pgconn = get_dbconn('idep') huc8df = gpd.GeoDataFrame.from_postgis(""" SELECT huc8, ST_Transform(simple_geom, %s) as geo from wbd_huc8 WHERE swat_use ORDER by huc8 """, pgconn, params=(PROJSTR,), index_col='huc8', geom_col='geo') hucs = huc8df.index.values years = range(1981, 2018) nc = netCDF4.Dataset("%s/%s_daily.nc" % (basedir, years[0])) # compute the affine ncaffine = Affine(nc.variables['lon'][1] - nc.variables['lon'][0], 0., nc.variables['lon'][0], 0., nc.variables['lat'][0] - nc.variables['lat'][1], nc.variables['lat'][-1]) czs = CachingZonalStats(ncaffine) nc.close() fps = [] for year in years: nc = netCDF4.Dataset("%s/%s_daily.nc" % (basedir, year)) basedate, timesz = get_basedate(nc) for i in tqdm(range(timesz), desc=str(year)): # date = basedate + datetime.timedelta(days=i) # keep array logic in top-down order tasmax = np.flipud(nc.variables['tmax'][i, :, :]) tasmin = np.flipud(nc.variables['tmin'][i, :, :]) pr = np.flipud(nc.variables['ppt'][i, :, :]) mytasmax = czs.gen_stats(tasmax, huc8df['geo']) mytasmin = czs.gen_stats(tasmin, huc8df['geo']) mypr = czs.gen_stats(pr, huc8df['geo']) for j, huc12 in enumerate(hucs): if i == 0 and year == years[0]: fps.append([open(('%s/precipitation/P%s.txt' ) % (outdir, huc12), 'w'), open(('%s/temperature/T%s.txt' ) % (outdir, huc12), 'w')]) fps[j][0].write("%s\n" % (basedate.strftime("%Y%m%d"), )) fps[j][1].write("%s\n" % (basedate.strftime("%Y%m%d"), )) fps[j][0].write(("%.1f\n" ) % (mypr[j], )) fps[j][1].write(("%.2f,%.2f\n" ) % (mytasmax[j], mytasmin[j])) for fp in fps: fp[0].close() fp[1].close()
def get_data(ctx): """Do the processing work, please""" pgconn = get_dbconn("postgis") states = gpd.GeoDataFrame.from_postgis( """ SELECT the_geom, state_abbr from states where state_abbr = %s """, pgconn, params=(ctx["state"], ), index_col="state_abbr", geom_col="the_geom", ) if states.empty: raise NoDataFound("No data was found.") with ncopen(iemre.get_daily_ncname(ctx["year"])) as nc: precip = nc.variables["p01d"] czs = CachingZonalStats(iemre.AFFINE) hasdata = np.zeros( (nc.dimensions["lat"].size, nc.dimensions["lon"].size)) czs.gen_stats(hasdata, states["the_geom"]) for nav in czs.gridnav: grid = np.ones((nav.ysz, nav.xsz)) grid[nav.mask] = 0.0 jslice = slice(nav.y0, nav.y0 + nav.ysz) islice = slice(nav.x0, nav.x0 + nav.xsz) hasdata[jslice, islice] = np.where(grid > 0, 1, hasdata[jslice, islice]) ctx["iowa"] = np.flipud(hasdata) ctx["iowapts"] = float(np.sum(np.where(hasdata > 0, 1, 0))) now = datetime.datetime(ctx["year"], 1, 1) now += datetime.timedelta(days=(ctx["period"] - 1)) ets = datetime.datetime(ctx["year"], 12, 31) today = datetime.datetime.now() if ets > today: ets = today - datetime.timedelta(days=1) ctx["days"] = [] rows = [] trailthres = ((ctx["trailthres"] * units("inch")).to( units("mm")).magnitude) daythres = (ctx["daythres"] * units("inch")).to(units("mm")).magnitude while now < ets: rows.append(do_date(ctx, now, precip, daythres, trailthres)) now += datetime.timedelta(days=1) return pd.DataFrame(rows)
def compute_hasdata(year): """Compute the has_data grid""" nc = ncopen(iemre.get_daily_ncname(year), 'a', timeout=300) czs = CachingZonalStats(iemre.AFFINE) pgconn = get_dbconn('postgis') states = gpd.GeoDataFrame.from_postgis(""" SELECT the_geom, state_abbr from states where state_abbr not in ('AK', 'HI') """, pgconn, index_col='state_abbr', geom_col='the_geom') data = np.flipud(nc.variables['hasdata'][:, :]) czs.gen_stats(data, states['the_geom']) for nav in czs.gridnav: grid = np.ones((nav.ysz, nav.xsz)) grid[nav.mask] = 0. jslice = slice(nav.y0, nav.y0 + nav.ysz) islice = slice(nav.x0, nav.x0 + nav.xsz) data[jslice, islice] = np.where(grid > 0, 1, data[jslice, islice]) nc.variables['hasdata'][:, :] = np.flipud(data) nc.close()
def get_data(ctx): """Do the processing work, please""" pgconn = get_dbconn('postgis') states = gpd.GeoDataFrame.from_postgis(""" SELECT the_geom, state_abbr from states where state_abbr = %s """, pgconn, params=(ctx['state'], ), index_col='state_abbr', geom_col='the_geom') with ncopen(iemre.get_daily_ncname(ctx['year'])) as nc: precip = nc.variables['p01d'] czs = CachingZonalStats(iemre.AFFINE) hasdata = np.zeros( (nc.dimensions['lat'].size, nc.dimensions['lon'].size)) czs.gen_stats(hasdata, states['the_geom']) for nav in czs.gridnav: grid = np.ones((nav.ysz, nav.xsz)) grid[nav.mask] = 0. jslice = slice(nav.y0, nav.y0 + nav.ysz) islice = slice(nav.x0, nav.x0 + nav.xsz) hasdata[jslice, islice] = np.where(grid > 0, 1, hasdata[jslice, islice]) ctx['iowa'] = np.flipud(hasdata) ctx['iowapts'] = float(np.sum(np.where(hasdata > 0, 1, 0))) now = datetime.datetime(ctx['year'], 1, 1) now += datetime.timedelta(days=(ctx['period'] - 1)) ets = datetime.datetime(ctx['year'], 12, 31) today = datetime.datetime.now() if ets > today: ets = today - datetime.timedelta(days=1) ctx['days'] = [] rows = [] trailthres = (ctx['trailthres'] * units('inch')).to( units('mm')).magnitude daythres = (ctx['daythres'] * units('inch')).to(units('mm')).magnitude while now < ets: rows.append(do_date(ctx, now, precip, daythres, trailthres)) now += datetime.timedelta(days=1) return pd.DataFrame(rows)
def compute_hasdata(): """Compute the has_data grid""" nc = ncopen(iemre.get_dailyc_ncname(), "a", timeout=300) czs = CachingZonalStats(iemre.AFFINE) pgconn = get_dbconn("postgis") states = gpd.GeoDataFrame.from_postgis( "SELECT the_geom, state_abbr from state " "where state_abbr not in ('AK', 'HI')", pgconn, index_col="state_abbr", geom_col="the_geom", ) data = np.flipud(nc.variables["hasdata"][:, :]) czs.gen_stats(data, states["the_geom"]) for nav in czs.gridnav: grid = np.ones((nav.ysz, nav.xsz)) grid[nav.mask] = 0.0 yslice = slice(nav.y0, nav.y0 + nav.ysz) xslice = slice(nav.x0, nav.x0 + nav.xsz) data[yslice, xslice] = np.where(grid > 0, 1, data[yslice, xslice]) nc.variables["hasdata"][:, :] = np.flipud(data) nc.close()
def do_day(valid): """ Process a day please """ idx = iemre.daily_offset(valid) nc = ncopen(iemre.get_daily_ncname(valid.year), 'r', timeout=300) high = temperature(nc.variables['high_tmpk_12z'][idx, :, :], 'K').value('F') low = temperature(nc.variables['low_tmpk_12z'][idx, :, :], 'K').value('F') precip = distance(nc.variables['p01d_12z'][idx, :, :], 'MM').value("IN") snow = distance(nc.variables['snow_12z'][idx, :, :], 'MM').value("IN") snowd = distance(nc.variables['snowd_12z'][idx, :, :], 'MM').value("IN") nc.close() # build out the state mappers pgconn = get_dbconn('postgis') states = gpd.GeoDataFrame.from_postgis(""" SELECT the_geom, state_abbr from states where state_abbr not in ('AK', 'HI', 'DC') """, pgconn, index_col='state_abbr', geom_col='the_geom') czs = CachingZonalStats(iemre.AFFINE) sthigh = czs.gen_stats(np.flipud(high), states['the_geom']) stlow = czs.gen_stats(np.flipud(low), states['the_geom']) stprecip = czs.gen_stats(np.flipud(precip), states['the_geom']) stsnow = czs.gen_stats(np.flipud(snow), states['the_geom']) stsnowd = czs.gen_stats(np.flipud(snowd), states['the_geom']) statedata = {} for i, state in enumerate(states.index.values): statedata[state] = dict(high=sthigh[i], low=stlow[i], precip=stprecip[i], snow=stsnow[i], snowd=stsnowd[i]) update_database(state + "0000", valid, statedata[state]) # build out climate division mappers climdiv = gpd.GeoDataFrame.from_postgis(""" SELECT geom, iemid from climdiv where st_abbrv not in ('AK', 'HI', 'DC') """, pgconn, index_col='iemid', geom_col='geom') czs = CachingZonalStats(iemre.AFFINE) sthigh = czs.gen_stats(np.flipud(high), climdiv['geom']) stlow = czs.gen_stats(np.flipud(low), climdiv['geom']) stprecip = czs.gen_stats(np.flipud(precip), climdiv['geom']) stsnow = czs.gen_stats(np.flipud(snow), climdiv['geom']) stsnowd = czs.gen_stats(np.flipud(snowd), climdiv['geom']) for i, iemid in enumerate(climdiv.index.values): row = dict(high=sthigh[i], low=stlow[i], precip=stprecip[i], snow=stsnow[i], snowd=stsnowd[i]) # we must have temperature data if row['high'] is np.ma.masked or row['low'] is np.ma.masked: print( ("compute_0000 %s has missing temperature data, using state") % (iemid, )) row = statedata[iemid[:2]] update_database(iemid, valid, row)
def plotter(fdict): """ Go """ ctx = get_autoplot_context(fdict, get_description()) csector = ctx['csector'] sdate = make_tuesday(ctx['sdate']) edate = make_tuesday(ctx['edate']) dlevel = ctx['d'] griddelta = 0.1 mp = MapPlot(sector=('state' if len(csector) == 2 else csector), state=ctx['csector'], title=('%s at or above "%s" %s - %s') % (PDICT2[ctx['w']], PDICT[dlevel], sdate.strftime("%b %-d, %Y"), edate.strftime("%b %-d, %Y")), subtitle=('based on weekly US Drought Monitor Analysis, ' '%.2f$^\circ$ grid analysis') % (griddelta, ), continentalcolor='white', titlefontsize=14) # compute the affine (west, east, south, north) = mp.ax.get_extent(ccrs.PlateCarree()) raster = np.zeros((int( (north - south) / griddelta), int((east - west) / griddelta))) lons = np.arange(raster.shape[1]) * griddelta + west lats = np.arange(0, 0 - raster.shape[0], -1) * griddelta + north lats = lats[::-1] affine = Affine(griddelta, 0., west, 0., 0 - griddelta, north) # get the geopandas data pgconn = get_dbconn('postgis') df = read_postgis(""" with d as ( select valid, (ST_Dump(st_simplify(geom, 0.01))).geom from usdm where valid >= %s and valid <= %s and dm >= %s and ST_Intersects(geom, ST_GeomFromEWKT('SRID=4326;POLYGON((%s %s, %s %s, %s %s, %s %s, %s %s))')) ) select valid, st_collect(geom) as the_geom from d GROUP by valid """, pgconn, params=(sdate, edate, dlevel, west, south, west, north, east, north, east, south, west, south), geom_col='the_geom') if df.empty: raise NoDataFound("No Data Found, sorry!") # loop over the cached stats czs = CachingZonalStats(affine) czs.compute_gridnav(df['the_geom'], raster) for nav in czs.gridnav: if nav is None: continue grid = np.ones((nav.ysz, nav.xsz)) grid[nav.mask] = 0. jslice = slice(nav.y0, nav.y0 + nav.ysz) islice = slice(nav.x0, nav.x0 + nav.xsz) raster[jslice, islice] += grid maxval = 10 if np.max(raster) < 11 else np.max(raster) ramp = np.linspace(1, maxval + 1, 11, dtype='i') if ctx['w'] == 'percent': ramp = np.arange(0, 101, 10) ramp[0] = 1. ramp[-1] = 100.1 # we add one since we are rectified to tuesdays, so we have an extra # week in there raster = raster / ((edate - sdate).days / 7. + 1.) * 100. # plot cmap = stretch_cmap(ctx['cmap'], ramp) cmap.set_under('white') cmap.set_bad('white') mp.pcolormesh(lons, lats, np.flipud(raster), ramp, cmap=cmap, units='count' if ctx['w'] == 'weeks' else 'Percent') if len(csector) == 2: mp.drawcounties() mp.drawcities() rows = [] for j in range(raster.shape[0]): for i in range(raster.shape[1]): rows.append(dict(lon=lons[i], lat=lats[j], value=raster[j, i])) return mp.fig, pd.DataFrame(rows)
def plotter(fdict): """ Go """ ctx = util.get_autoplot_context(fdict, get_description()) date = ctx['date'] sector = ctx['sector'] days = ctx['trailing'] threshold = ctx['threshold'] window_sts = date - datetime.timedelta(days=days) if window_sts.year != date.year: raise ValueError('Sorry, do not support multi-year plots yet!') if len(sector) != 2: raise ValueError("Sorry, this does not support multi-state plots yet.") idx0 = iemre.daily_offset(window_sts) idx1 = iemre.daily_offset(date) ncfn = iemre.get_daily_mrms_ncname(date.year) ncvar = 'p01d' if not os.path.isfile(ncfn): raise ValueError("No data for that year, sorry.") nc = util.ncopen(ncfn) # Get the state weight df = gpd.GeoDataFrame.from_postgis(""" SELECT the_geom from states where state_abbr = %s """, util.get_dbconn('postgis'), params=(sector, ), index_col=None, geom_col='the_geom') czs = CachingZonalStats(iemre.MRMS_AFFINE) czs.gen_stats( np.zeros((nc.variables['lat'].size, nc.variables['lon'].size)), df['the_geom']) hasdata = None jslice = None islice = None for nav in czs.gridnav: hasdata = np.ones((nav.ysz, nav.xsz)) hasdata[nav.mask] = 0. # careful here as y is flipped in this context jslice = slice(nc.variables['lat'].size - (nav.y0 + nav.ysz), nc.variables['lat'].size - nav.y0) islice = slice(nav.x0, nav.x0 + nav.xsz) hasdata = np.flipud(hasdata) today = distance(nc.variables[ncvar][idx1, jslice, islice], 'MM').value('IN') if (idx1 - idx0) < 32: p01d = distance( np.sum(nc.variables[ncvar][idx0:idx1, jslice, islice], 0), 'MM').value('IN') else: # Too much data can overwhelm this app, need to chunk it for i in range(idx0, idx1, 10): i2 = min([i + 10, idx1]) if idx0 == i: p01d = distance( np.sum(nc.variables[ncvar][i:i2, jslice, islice], 0), 'MM').value('IN') else: p01d += distance( np.sum(nc.variables[ncvar][i:i2, jslice, islice], 0), 'MM').value('IN') nc.close() # Get climatology nc = util.ncopen(iemre.get_dailyc_mrms_ncname()) if (idx1 - idx0) < 32: c_p01d = distance( np.sum(nc.variables[ncvar][idx0:idx1, jslice, islice], 0), 'MM').value('IN') else: # Too much data can overwhelm this app, need to chunk it for i in range(idx0, idx1, 10): i2 = min([i + 10, idx1]) if idx0 == i: c_p01d = distance( np.sum(nc.variables[ncvar][i:i2, jslice, islice], 0), 'MM').value('IN') else: c_p01d += distance( np.sum(nc.variables[ncvar][i:i2, jslice, islice], 0), 'MM').value('IN') nc.close() # we actually don't care about weights at this fine of scale cells = np.sum(np.where(hasdata > 0, 1, 0)) departure = p01d - c_p01d # Update departure and today to values unconsidered below when out of state departure = np.where(hasdata > 0, departure, -9999) today = np.where(hasdata > 0, today, 0) ranges = [[-99, -3], [-3, -2], [-2, -1], [-1, 0], [0, 1], [1, 2], [2, 3], [3, 99]] x = [] x2 = [] labels = [] for (minv, maxv) in ranges: labels.append("%.0f to %.0f" % (minv, maxv)) # How many departure cells in this range hits = np.logical_and(departure < maxv, departure > minv) hits2 = np.logical_and(hits, today > threshold) x.append(np.sum(np.where(hits, 1, 0)) / float(cells) * 100.) x2.append(np.sum(np.where(hits2, 1, 0)) / float(cells) * 100.) (fig, ax) = plt.subplots(1, 1) ax.set_title(("%s NOAA MRMS %s %.2f inch Precip Coverage") % (state_names[sector], date.strftime("%-d %b %Y"), threshold)) ax.bar(np.arange(8) - 0.2, x, align='center', width=0.4, label='Trailing %s Day Departure' % (days, )) ax.bar(np.arange(8) + 0.2, x2, align='center', width=0.4, label='%s Coverage (%.1f%% Tot)' % (date.strftime("%-d %b %Y"), sum(x2))) for i, (_x1, _x2) in enumerate(zip(x, x2)): ax.text(i - 0.2, _x1 + 1, "%.1f" % (_x1, ), ha='center') ax.text(i + 0.2, _x2 + 1, "%.1f" % (_x2, ), ha='center') ax.set_xticks(np.arange(8)) ax.set_xticklabels(labels) ax.set_xlabel("Trailing %s Day Precip Departure [in]" % (days, )) ax.set_position([0.1, 0.2, 0.8, 0.7]) ax.legend(loc=(0., -0.2), ncol=2) ax.set_ylabel("Areal Coverage of %s [%%]" % (state_names[sector], )) ax.grid(True) ax.set_xlim(-0.5, 7.5) ax.set_ylim(0, max([max(x2), max(x)]) + 5) return fig
def plotter(fdict): """ Go """ ctx = util.get_autoplot_context(fdict, get_description()) date = ctx['date'] sector = ctx['sector'] threshold = ctx['threshold'] threshold_mm = distance(threshold, 'IN').value('MM') window_sts = date - datetime.timedelta(days=90) if window_sts.year != date.year: raise NoDataFound('Sorry, do not support multi-year plots yet!') # idx0 = iemre.daily_offset(window_sts) idx1 = iemre.daily_offset(date) ncfn = iemre.get_daily_mrms_ncname(date.year) if not os.path.isfile(ncfn): raise NoDataFound("No data found.") ncvar = 'p01d' # Get the state weight df = gpd.GeoDataFrame.from_postgis(""" SELECT the_geom from states where state_abbr = %s """, util.get_dbconn('postgis'), params=(sector, ), index_col=None, geom_col='the_geom') czs = CachingZonalStats(iemre.MRMS_AFFINE) with util.ncopen(ncfn) as nc: czs.gen_stats( np.zeros((nc.variables['lat'].size, nc.variables['lon'].size)), df['the_geom']) jslice = None islice = None for nav in czs.gridnav: # careful here as y is flipped in this context jslice = slice(nc.variables['lat'].size - (nav.y0 + nav.ysz), nc.variables['lat'].size - nav.y0) islice = slice(nav.x0, nav.x0 + nav.xsz) grid = np.zeros( (jslice.stop - jslice.start, islice.stop - islice.start)) total = np.zeros( (jslice.stop - jslice.start, islice.stop - islice.start)) for i, idx in enumerate(range(idx1, idx1 - 90, -1)): total += nc.variables[ncvar][idx, jslice, islice] grid = np.where(np.logical_and(grid == 0, total > threshold_mm), i, grid) lon = nc.variables['lon'][islice] lat = nc.variables['lat'][jslice] mp = MapPlot(sector='state', state=sector, titlefontsize=14, subtitlefontsize=12, title=("NOAA MRMS Q3: Number of Recent Days " "till Accumulating %s\" of Precip") % (threshold, ), subtitle=("valid %s: based on per calendar day " "estimated preciptation, GaugeCorr and " "RadarOnly products") % (date.strftime("%-d %b %Y"), )) x, y = np.meshgrid(lon, lat) cmap = plt.get_cmap(ctx['cmap']) cmap.set_over('k') cmap.set_under('white') mp.pcolormesh(x, y, grid, np.arange(0, 81, 10), cmap=cmap, units='days') mp.drawcounties() mp.drawcities() return mp.fig
def plotter(fdict): """ Go """ ctx = get_autoplot_context(fdict, get_description()) year = ctx['year'] threshold = ctx['threshold'] period = ctx['period'] state = ctx['state'] pgconn = get_dbconn('postgis') states = gpd.GeoDataFrame.from_postgis(""" SELECT the_geom, state_abbr from states where state_abbr = %s """, pgconn, params=(state, ), index_col='state_abbr', geom_col='the_geom') nc = ncopen(iemre.get_daily_ncname(year)) precip = nc.variables['p01d'] czs = CachingZonalStats(iemre.AFFINE) hasdata = np.zeros((nc.dimensions['lat'].size, nc.dimensions['lon'].size)) czs.gen_stats(hasdata, states['the_geom']) for nav in czs.gridnav: grid = np.ones((nav.ysz, nav.xsz)) grid[nav.mask] = 0. jslice = slice(nav.y0, nav.y0 + nav.ysz) islice = slice(nav.x0, nav.x0 + nav.xsz) hasdata[jslice, islice] = np.where(grid > 0, 1, hasdata[jslice, islice]) hasdata = np.flipud(hasdata) datapts = np.sum(np.where(hasdata > 0, 1, 0)) now = datetime.date(year, 1, 1) now += datetime.timedelta(days=(period - 1)) ets = datetime.date(year, 12, 31) today = datetime.date.today() if ets > today: ets = today days = [] coverage = [] while now <= ets: idx = iemre.daily_offset(now) sevenday = np.sum(precip[(idx - period):idx, :, :], 0) pday = np.where(hasdata > 0, sevenday[:, :], -1) tots = np.sum(np.where(pday >= (threshold * 25.4), 1, 0)) days.append(now) coverage.append(tots / float(datapts) * 100.0) now += datetime.timedelta(days=1) df = pd.DataFrame(dict(day=pd.Series(days), coverage=pd.Series(coverage))) (fig, ax) = plt.subplots(1, 1) ax.bar(days, coverage, fc='g', ec='g') ax.set_title( ("%s IEM Estimated Areal Coverage Percent of %s\n" " receiving %.2f inches of rain over trailing %s day period") % (year, reference.state_names[state], threshold, period)) ax.set_ylabel("Areal Coverage [%]") ax.xaxis.set_major_formatter(mdates.DateFormatter('%b\n%-d')) ax.set_yticks(range(0, 101, 25)) ax.grid(True) return fig, df
def plotter(fdict): """ Go """ ctx = get_autoplot_context(fdict, get_description()) year = ctx['year'] thres = ctx['thres'] metric = distance(thres, 'IN').value('MM') state = ctx['state'][:2] sts = datetime.datetime(year, 10, 1) ets = datetime.datetime(year + 1, 5, 1) rows = [] pgconn = get_dbconn('postgis') states = gpd.GeoDataFrame.from_postgis(""" SELECT the_geom, state_abbr from states where state_abbr = %s """, pgconn, params=(state, ), index_col='state_abbr', geom_col='the_geom') sidx = iemre.daily_offset(sts) ncfn = iemre.get_daily_ncname(sts.year) if not os.path.isfile(ncfn): raise ValueError("Data for year %s not found" % (sts.year, )) nc = ncopen(ncfn) czs = CachingZonalStats(iemre.AFFINE) hasdata = np.zeros((nc.dimensions['lat'].size, nc.dimensions['lon'].size)) czs.gen_stats(hasdata, states['the_geom']) for nav in czs.gridnav: grid = np.ones((nav.ysz, nav.xsz)) grid[nav.mask] = 0. jslice = slice(nav.y0, nav.y0 + nav.ysz) islice = slice(nav.x0, nav.x0 + nav.xsz) hasdata[jslice, islice] = np.where(grid > 0, 1, hasdata[jslice, islice]) st = np.flipud(hasdata) stpts = np.sum(np.where(hasdata > 0, 1, 0)) snowd = nc.variables['snowd_12z'][sidx:, :, :] nc.close() for i in range(snowd.shape[0]): rows.append({ 'valid': sts + datetime.timedelta(days=i), 'coverage': f(st, snowd[i], metric, stpts), }) eidx = iemre.daily_offset(ets) nc = ncopen(iemre.get_daily_ncname(ets.year)) snowd = nc.variables['snowd_12z'][:eidx, :, :] nc.close() for i in range(snowd.shape[0]): rows.append({ 'valid': datetime.date(ets.year, 1, 1) + datetime.timedelta(days=i), 'coverage': f(st, snowd[i], metric, stpts), }) df = pd.DataFrame(rows) df = df[np.isfinite(df['coverage'])] (fig, ax) = plt.subplots(1, 1, sharex=True, figsize=(8, 6)) ax.bar(df['valid'].values, df['coverage'].values, fc='tan', ec='tan', align='center') ax.set_title(("IEM Estimated Areal Snow Coverage Percent of %s\n" " percentage of state reporting at least %.2fin snow" " cover" ) % (reference.state_names[state], thres)) ax.set_ylabel("Areal Coverage [%]") ax.xaxis.set_major_locator(mdates.DayLocator([1, 15])) ax.xaxis.set_major_formatter(mdates.DateFormatter("%-d %b\n%Y")) ax.set_yticks(range(0, 101, 25)) ax.grid(True) return fig, df
def main(): """Go Main Go""" basedir = "/mnt/nrel/akrherz/livneh" outdir = "swatfiles_livneh" if os.path.isdir(outdir): print("ABORT: as %s exists" % (outdir, )) return os.mkdir(outdir) for dirname in ['precipitation', 'temperature']: os.mkdir("%s/%s" % (outdir, dirname)) pgconn = get_dbconn('idep') huc12df = gpd.GeoDataFrame.from_postgis(""" SELECT huc12, ST_Transform(simple_geom, %s) as geo from wbd_huc12 WHERE swat_use ORDER by huc12 """, pgconn, params=(PROJSTR,), index_col='huc12', geom_col='geo') hucs = huc12df.index.values years = range(1989, 2011) nc = ncopen("%s/livneh_NAmerExt_15Oct2014.198109.nc" % (basedir, )) # compute the affine ncaffine = Affine(nc.variables['lon'][1] - nc.variables['lon'][0], 0., nc.variables['lon'][0], 0., nc.variables['lat'][0] - nc.variables['lat'][1], nc.variables['lat'][-1]) czs = CachingZonalStats(ncaffine) nc.close() fps = [] for year in years: for month in range(1, 13): nc = ncopen(("%s/livneh_NAmerExt_15Oct2014.%s%02i.nc" ) % (basedir, year, month)) basedate, timesz = get_basedate(nc) for i in tqdm(range(timesz), desc="%s%02i" % (year, month)): tasmax = np.flipud(nc.variables['Tmax'][i, :, :]) tasmin = np.flipud(nc.variables['Tmin'][i, :, :]) pr = np.flipud(nc.variables['Prec'][i, :, :]) mytasmax = czs.gen_stats(tasmax, huc12df['geo']) mytasmin = czs.gen_stats(tasmin, huc12df['geo']) mypr = czs.gen_stats(pr, huc12df['geo']) for j, huc12 in enumerate(hucs): if i == 0 and year == years[0] and month == 1: fps.append([open(('%s/precipitation/P%s.txt' ) % (outdir, huc12), 'w'), open(('%s/temperature/T%s.txt' ) % (outdir, huc12), 'w')]) fps[j][0].write( "%s\n" % (basedate.strftime("%Y%m%d"), )) fps[j][1].write( "%s\n" % (basedate.strftime("%Y%m%d"), )) fps[j][0].write(("%.1f\n" ) % (mypr[j] if mypr[j] is not None else 0, )) fps[j][1].write(("%.2f,%.2f\n" ) % (mytasmax[j], mytasmin[j])) for fp in fps: fp[0].close() fp[1].close()
def main(argv): """Go Main Go""" model = argv[1] rcp = argv[2] basedir = "/mnt/nrel/akrherz/loca/%s/16th/%s/r1i1p1" % (model, rcp) outdir = "swatfiles_%s_%s" % (model, rcp) if os.path.isdir(outdir): print("ABORT: as %s exists" % (outdir, )) return os.mkdir(outdir) for dirname in ['precipitation', 'temperature']: os.mkdir("%s/%s" % (outdir, dirname)) pgconn = get_dbconn('idep') huc12df = gpd.GeoDataFrame.from_postgis(""" SELECT huc12, ST_Transform(simple_geom, %s) as geo from wbd_huc12 WHERE swat_use ORDER by huc12 """, pgconn, params=(PROJSTR,), index_col='huc12', geom_col='geo') hucs = huc12df.index.values years = range(1989, 2011) if rcp == 'historical' else range(2039, 2061) nc = netCDF4.Dataset(("%s/pr/pr_day_%s_%s_r1i1p1" "_%.0f0101-%.0f1231.LOCA_2016-04-02.16th.nc" ) % (basedir, model, rcp, years[0], years[0])) # compute the affine ncaffine = Affine(nc.variables['lon'][1] - nc.variables['lon'][0], 0., nc.variables['lon'][0] - 360., 0., nc.variables['lat'][0] - nc.variables['lat'][1], nc.variables['lat'][-1]) czs = CachingZonalStats(ncaffine) nc.close() fps = [] for year in years: # assume 2006-2010 is closely represented by rcp45 if year >= 2006 and year < 2011: rcp = 'rcp45' basedir = "/mnt/nrel/akrherz/loca/%s/16th/%s/r1i1p1" % (model, rcp) pr_nc = netCDF4.Dataset(("%s/pr/pr_day_%s_%s_r1i1p1" "_%.0f0101-%.0f1231.LOCA_2016-04-02.16th.nc" ) % (basedir, model, rcp, year, year)) tasmax_nc = netCDF4.Dataset(("%s/tasmax/tasmax_day_%s_%s_r1i1p1" "_%.0f0101-%.0f1231.LOCA_" "2016-04-02.16th.nc" ) % (basedir, model, rcp, year, year)) tasmin_nc = netCDF4.Dataset(("%s/tasmin/tasmin_day_%s_%s_r1i1p1" "_%.0f0101-%.0f1231.LOCA_" "2016-04-02.16th.nc" ) % (basedir, model, rcp, year, year)) basedate, timesz = get_basedate(pr_nc) for i in tqdm(range(timesz), desc=str(year)): # date = basedate + datetime.timedelta(days=i) # keep array logic in top-down order tasmax = np.flipud( temperature(tasmax_nc.variables['tasmax'][i, :, :], 'K').value('C')) tasmin = np.flipud( temperature(tasmin_nc.variables['tasmin'][i, :, :], 'K').value('C')) pr = np.flipud(pr_nc.variables['pr'][i, :, :]) mytasmax = czs.gen_stats(tasmax, huc12df['geo']) mytasmin = czs.gen_stats(tasmin, huc12df['geo']) mypr = czs.gen_stats(pr, huc12df['geo']) for j, huc12 in enumerate(hucs): if i == 0 and year == years[0]: fps.append([open(('%s/precipitation/P%s.txt' ) % (outdir, huc12), 'w'), open(('%s/temperature/T%s.txt' ) % (outdir, huc12), 'w')]) fps[j][0].write("%s\n" % (basedate.strftime("%Y%m%d"), )) fps[j][1].write("%s\n" % (basedate.strftime("%Y%m%d"), )) fps[j][0].write(("%.1f\n" ) % (mypr[j] * 86400., )) fps[j][1].write(("%.2f,%.2f\n" ) % (mytasmax[j], mytasmin[j])) for fp in fps: fp[0].close() fp[1].close()
def do_day(valid): """ Process a day please """ idx = iemre.daily_offset(valid) with ncopen(iemre.get_daily_ncname(valid.year), "r", timeout=300) as nc: high = temperature( nc.variables["high_tmpk_12z"][idx, :, :], "K" ).value("F") low = temperature(nc.variables["low_tmpk_12z"][idx, :, :], "K").value( "F" ) precip = distance(nc.variables["p01d_12z"][idx, :, :], "MM").value( "IN" ) snow = distance(nc.variables["snow_12z"][idx, :, :], "MM").value("IN") snowd = distance(nc.variables["snowd_12z"][idx, :, :], "MM").value( "IN" ) # build out the state mappers pgconn = get_dbconn("postgis") states = gpd.GeoDataFrame.from_postgis( """ SELECT the_geom, state_abbr from states where state_abbr not in ('AK', 'HI', 'DC') """, pgconn, index_col="state_abbr", geom_col="the_geom", ) czs = CachingZonalStats(iemre.AFFINE) sthigh = czs.gen_stats(np.flipud(high), states["the_geom"]) stlow = czs.gen_stats(np.flipud(low), states["the_geom"]) stprecip = czs.gen_stats(np.flipud(precip), states["the_geom"]) stsnow = czs.gen_stats(np.flipud(snow), states["the_geom"]) stsnowd = czs.gen_stats(np.flipud(snowd), states["the_geom"]) statedata = {} for i, state in enumerate(states.index.values): statedata[state] = dict( high=sthigh[i], low=stlow[i], precip=stprecip[i], snow=stsnow[i], snowd=stsnowd[i], ) update_database(state + "0000", valid, statedata[state]) # build out climate division mappers climdiv = gpd.GeoDataFrame.from_postgis( """ SELECT geom, iemid from climdiv where st_abbrv not in ('AK', 'HI', 'DC') """, pgconn, index_col="iemid", geom_col="geom", ) czs = CachingZonalStats(iemre.AFFINE) sthigh = czs.gen_stats(np.flipud(high), climdiv["geom"]) stlow = czs.gen_stats(np.flipud(low), climdiv["geom"]) stprecip = czs.gen_stats(np.flipud(precip), climdiv["geom"]) stsnow = czs.gen_stats(np.flipud(snow), climdiv["geom"]) stsnowd = czs.gen_stats(np.flipud(snowd), climdiv["geom"]) for i, iemid in enumerate(climdiv.index.values): row = dict( high=sthigh[i], low=stlow[i], precip=stprecip[i], snow=stsnow[i], snowd=stsnowd[i], ) # we must have temperature data if row["high"] is np.ma.masked or row["low"] is np.ma.masked: print( ("compute_0000 %s has missing temperature data, using state") % (iemid,) ) row = statedata[iemid[:2]] update_database(iemid, valid, row)