def do_precip12(ts): """Compute the 24 Hour precip at 12 UTC, we do some more tricks though""" offset = iemre.daily_offset(ts) ets = utc(ts.year, ts.month, ts.day, 12) sts = ets - datetime.timedelta(hours=24) offset1 = iemre.hourly_offset(sts) offset2 = iemre.hourly_offset(ets) if ts.month == 1 and ts.day == 1: if sts.year >= 1900: print(("p01d_12z for %s [idx:%s] %s(%s)->%s(%s) SPECIAL") % (ts, offset, sts.strftime("%Y%m%d%H"), offset1, ets.strftime("%Y%m%d%H"), offset2)) ncfn = iemre.get_hourly_ncname(ets.year) if not os.path.isfile(ncfn): print("Missing %s" % (ncfn, )) return hnc = ncopen(ncfn, timeout=600) phour = np.sum(hnc.variables['p01m'][:offset2, :, :], 0) hnc.close() hnc = ncopen(iemre.get_hourly_ncname(sts.year), timeout=600) phour += np.sum(hnc.variables['p01m'][offset1:, :, :], 0) hnc.close() else: ncfn = iemre.get_hourly_ncname(ts.year) if not os.path.isfile(ncfn): print("Missing %s" % (ncfn, )) return hnc = ncopen(ncfn, timeout=600) phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0) hnc.close() write_grid(ts, 'p01d_12z', np.where(phour < 0, 0, phour))
def do_precip12(nc, ts): """Compute the 24 Hour precip at 12 UTC, we do some more tricks though""" offset = iemre.daily_offset(ts) ets = datetime.datetime(ts.year, ts.month, ts.day, 12) ets = ets.replace(tzinfo=pytz.timezone("UTC")) sts = ets - datetime.timedelta(hours=24) offset1 = iemre.hourly_offset(sts) offset2 = iemre.hourly_offset(ets) if ts.month == 1 and ts.day == 1: print(("p01d_12z for %s [idx:%s] %s(%s)->%s(%s) SPECIAL" ) % (ts, offset, sts.strftime("%Y%m%d%H"), offset1, ets.strftime("%Y%m%d%H"), offset2)) ncfn = "/mesonet/data/iemre/%s_mw_hourly.nc" % (ets.year,) if not os.path.isfile(ncfn): print("Missing %s" % (ncfn,)) return hnc = netCDF4.Dataset(ncfn) phour = np.sum(hnc.variables['p01m'][:offset2, :, :], 0) hnc.close() hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % ( sts.year,)) phour += np.sum(hnc.variables['p01m'][offset1:, :, :], 0) hnc.close() else: print(("p01d_12z for %s [idx:%s] %s(%s)->%s(%s)" ) % (ts, offset, sts.strftime("%Y%m%d%H"), offset1, ets.strftime("%Y%m%d%H"), offset2)) ncfn = "/mesonet/data/iemre/%s_mw_hourly.nc" % (ts.year,) if not os.path.isfile(ncfn): print("Missing %s" % (ncfn,)) return hnc = netCDF4.Dataset(ncfn) phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0) hnc.close() nc.variables['p01d_12z'][offset] = phour
def dowork(form): """Do work!""" date = datetime.datetime.strptime(form.getfirst('valid'), '%Y-%m-%d') lat = float(form.getfirst("lat")) lon = float(form.getfirst("lon")) # We want data for the UTC date and timestamps are in the rears, so from # 1z through 1z sts = datetime.datetime(date.year, date.month, date.day, 1) sts = sts.replace(tzinfo=pytz.utc) ets = sts + datetime.timedelta(hours=24) sidx = iemre.hourly_offset(sts) eidx = iemre.hourly_offset(ets) ncfn = "/mesonet/data/stage4/%s_stage4_hourly.nc" % (date.year, ) nc = netCDF4.Dataset(ncfn, 'r') dist = ((nc.variables['lon'][:] - lon)**2 + (nc.variables['lat'][:] - lat)**2)**0.5 (j, i) = np.unravel_index(dist.argmin(), dist.shape) res = {'gridi': i, 'gridj': j, 'data': []} ppt = nc.variables['p01m'][sidx:eidx, j, i] nc.close() for tx, pt in enumerate(ppt): valid = sts + datetime.timedelta(hours=tx) res['data'].append({ 'end_valid': valid.strftime("%Y-%m-%dT%H:00:00Z"), 'precip_in': myrounder(datatypes.distance(pt, 'MM').value('IN'), 2) }) return json.dumps(res)
def do_precip12(ts, ds): """Compute the 24 Hour precip at 12 UTC, we do some more tricks though""" offset = iemre.daily_offset(ts) ets = utc(ts.year, ts.month, ts.day, 12) sts = ets - datetime.timedelta(hours=24) offset1 = iemre.hourly_offset(sts) offset2 = iemre.hourly_offset(ets) if ts.month == 1 and ts.day == 1: if sts.year >= 1900: LOG.warning( "p01d_12z for %s [idx:%s] %s(%s)->%s(%s) SPECIAL", ts, offset, sts.strftime("%Y%m%d%H"), offset1, ets.strftime("%Y%m%d%H"), offset2, ) ncfn = iemre.get_hourly_ncname(ets.year) if not os.path.isfile(ncfn): LOG.warning("Missing %s", ncfn) return with ncopen(ncfn, timeout=600) as hnc: phour = np.sum(hnc.variables["p01m"][:offset2, :, :], 0) with ncopen(iemre.get_hourly_ncname(sts.year), timeout=600) as hnc: phour += np.sum(hnc.variables["p01m"][offset1:, :, :], 0) else: ncfn = iemre.get_hourly_ncname(ts.year) if not os.path.isfile(ncfn): LOG.warning("Missing %s", ncfn) return with ncopen(ncfn, timeout=600) as hnc: phour = np.sum(hnc.variables["p01m"][offset1:offset2, :, :], 0) ds["p01d_12z"].values = np.where(phour < 0, 0, phour)
def do_precip(nc, ts): """Compute the 6 UTC to 6 UTC precip We need to be careful here as the timestamp sent to this app is today, we are actually creating the analysis for yesterday """ sts = datetime.datetime(ts.year, ts.month, ts.day, 6) sts = sts.replace(tzinfo=pytz.timezone("UTC")) ets = sts + datetime.timedelta(hours=24) offset = iemre.daily_offset(ts) offset1 = iemre.hourly_offset(sts) offset2 = iemre.hourly_offset(ets) if ts.month == 12 and ts.day == 31: print(("p01d for %s [idx:%s] %s(%s)->%s(%s) SPECIAL") % (ts, offset, sts.strftime("%Y%m%d%H"), offset1, ets.strftime("%Y%m%d%H"), offset2)) hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (ets.year, )) phour = np.sum(hnc.variables['p01m'][:offset2, :, :], 0) hnc.close() hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (sts.year, )) phour += np.sum(hnc.variables['p01m'][offset1:, :, :], 0) hnc.close() else: print(("p01d for %s [idx:%s] %s(%s)->%s(%s)") % (ts, offset, sts.strftime("%Y%m%d%H"), offset1, ets.strftime("%Y%m%d%H"), offset2)) hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (sts.year, )) phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0) hnc.close() nc.variables['p01d'][offset] = phour
def do_precip(nc, ts): """Compute the 6 UTC to 6 UTC precip We need to be careful here as the timestamp sent to this app is today, we are actually creating the analysis for yesterday """ sts = datetime.datetime(ts.year, ts.month, ts.day, 6) sts = sts.replace(tzinfo=pytz.timezone("UTC")) ets = sts + datetime.timedelta(hours=24) offset = iemre.daily_offset(ts) offset1 = iemre.hourly_offset(sts) offset2 = iemre.hourly_offset(ets) if ts.month == 12 and ts.day == 31: print(("p01d for %s [idx:%s] %s(%s)->%s(%s) SPECIAL" ) % (ts, offset, sts.strftime("%Y%m%d%H"), offset1, ets.strftime("%Y%m%d%H"), offset2)) hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % ( ets.year,)) phour = np.sum(hnc.variables['p01m'][:offset2, :, :], 0) hnc.close() hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % ( sts.year,)) phour += np.sum(hnc.variables['p01m'][offset1:, :, :], 0) hnc.close() else: print(("p01d for %s [idx:%s] %s(%s)->%s(%s)" ) % (ts, offset, sts.strftime("%Y%m%d%H"), offset1, ets.strftime("%Y%m%d%H"), offset2)) hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % ( sts.year,)) phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0) hnc.close() nc.variables['p01d'][offset] = phour
def do_precip12(nc, ts): """Compute the 24 Hour precip at 12 UTC, we do some more tricks though""" offset = iemre.daily_offset(ts) ets = datetime.datetime(ts.year, ts.month, ts.day, 12) ets = ets.replace(tzinfo=pytz.timezone("UTC")) sts = ets - datetime.timedelta(hours=24) offset1 = iemre.hourly_offset(sts) offset2 = iemre.hourly_offset(ets) if ts.month == 1 and ts.day == 1: print(("p01d_12z for %s [idx:%s] %s(%s)->%s(%s) SPECIAL") % (ts, offset, sts.strftime("%Y%m%d%H"), offset1, ets.strftime("%Y%m%d%H"), offset2)) hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (ets.year, )) phour = np.sum(hnc.variables['p01m'][:offset2, :, :], 0) hnc.close() hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (sts.year, )) phour += np.sum(hnc.variables['p01m'][offset1:, :, :], 0) hnc.close() else: print(("p01d_12z for %s [idx:%s] %s(%s)->%s(%s)") % (ts, offset, sts.strftime("%Y%m%d%H"), offset1, ets.strftime("%Y%m%d%H"), offset2)) hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (ts.year, )) phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0) hnc.close() nc.variables['p01d_12z'][offset] = phour
def dowork(form): """Do work!""" date = datetime.datetime.strptime(form.getfirst('valid'), '%Y-%m-%d') lat = float(form.getfirst("lat")) lon = float(form.getfirst("lon")) # We want data for the UTC date and timestamps are in the rears, so from # 1z through 1z sts = utc(date.year, date.month, date.day, 1) ets = sts + datetime.timedelta(hours=24) sidx = iemre.hourly_offset(sts) eidx = iemre.hourly_offset(ets) ncfn = "/mesonet/data/stage4/%s_stage4_hourly.nc" % (date.year, ) res = {'gridi': -1, 'gridj': -1, 'data': []} if not os.path.isfile(ncfn): return json.dumps(res) with ncopen(ncfn) as nc: dist = ((nc.variables['lon'][:] - lon)**2 + (nc.variables['lat'][:] - lat)**2)**0.5 (j, i) = np.unravel_index(dist.argmin(), dist.shape) res['gridi'] = int(i) res['gridj'] = int(j) ppt = nc.variables['p01m'][sidx:eidx, j, i] for tx, pt in enumerate(ppt): valid = sts + datetime.timedelta(hours=tx) res['data'].append({ 'end_valid': valid.strftime("%Y-%m-%dT%H:00:00Z"), 'precip_in': myrounder( datatypes.distance(pt, 'MM').value('IN'), 2) }) return json.dumps(res)
def workflow(valid): """Our workflow""" if valid.month == 1 and valid.day == 1: print("prism_adjust_stage4, sorry Jan 1 processing is a TODO!") return # read prism tidx = daily_offset(valid) nc = netCDF4.Dataset("/mesonet/data/prism/%s_daily.nc" % (valid.year, ), 'r') ppt = nc.variables['ppt'][tidx, :, :] # missing as zero ppt = np.where(ppt.mask, 0, ppt) lons = nc.variables['lon'][:] lats = nc.variables['lat'][:] nc.close() (lons, lats) = np.meshgrid(lons, lats) # Interpolate this onto the stage4 grid nc = netCDF4.Dataset( ("/mesonet/data/stage4/%s_stage4_hourly.nc") % (valid.year, ), 'a') p01m = nc.variables['p01m'] p01m_status = nc.variables['p01m_status'] s4lons = nc.variables['lon'][:] s4lats = nc.variables['lat'][:] # Values are in the hourly arrears, so start at -23 and thru current hour sts_tidx = hourly_offset(valid - datetime.timedelta(hours=23)) ets_tidx = hourly_offset(valid + datetime.timedelta(hours=1)) s4total = np.sum(p01m[sts_tidx:ets_tidx, :, :], axis=0) # make sure the s4total does not have zeros s4total = np.where(s4total < 0.001, 0.001, s4total) nn = NearestNDInterpolator((lons.flat, lats.flat), ppt.flat) prism_on_s4grid = nn(s4lons, s4lats) multiplier = prism_on_s4grid / s4total # Do the work now, we should not have to worry about the scale factor for tidx in range(sts_tidx, ets_tidx): newval = p01m[tidx, :, :] * multiplier p01m[tidx, :, :] = newval # make sure have data if np.ma.max(newval) > 0: p01m_status[tidx] = 2 else: print(("prism_adjust_stage4 NOOP for time %s[idx:%s]") % ((datetime.datetime(valid.year, 1, 1, 0) + datetime.timedelta(hours=tidx)).strftime("%Y-%m-%dT%H"), tidx)) """ s4total_v2 = np.sum(p01m[sts_tidx:ets_tidx, :, :], axis=0) from pyiem.plot.geoplot import MapPlot import matplotlib.pyplot as plt mp = MapPlot(sector='iowa') mp.pcolormesh(s4lons, s4lats, s4total_v2, np.arange(-10, 11, 1), cmap=plt.get_cmap("BrBG")) mp.postprocess(filename='test.png') mp.close() """ nc.close()
def load_stage4(): """ It sucks, but we need to load the stage IV data to give us something to benchmark the MRMS data against, to account for two things: 1) Wind Farms 2) Over-estimates """ printt("load_stage4() called...") # The stage4 files store precip in the rears, so compute 1 AM one_am = datetime.datetime(VALID.year, VALID.month, VALID.day, 12, 0) one_am = one_am.replace(tzinfo=pytz.timezone("UTC")) one_am = one_am.astimezone(pytz.timezone("America/Chicago")) # stage4 data is stored in the rears, so 1am one_am = one_am.replace(hour=1, minute=0, second=0) # clever hack for CST/CDT tomorrow = one_am + datetime.timedelta(hours=36) tomorrow = tomorrow.replace(hour=1) sts_tidx = iemre.hourly_offset(one_am) ets_tidx = iemre.hourly_offset(tomorrow) printt(("stage4 sts_tidx:%s[%s] ets_tidx:%s[%s]" ) % (sts_tidx, one_am, ets_tidx, tomorrow)) nc = ncopen("/mesonet/data/stage4/%s_stage4_hourly.nc" % (VALID.year, )) p01m = nc.variables['p01m'] lats = nc.variables['lat'][:] lons = nc.variables['lon'][:] # crossing jan 1 if ets_tidx < sts_tidx: printt("Exercise special stageIV logic for jan1!") totals = np.sum(p01m[sts_tidx:, :, :], axis=0) nc.close() nc = ncopen( "/mesonet/data/stage4/%s_stage4_hourly.nc" % (tomorrow.year, ) ) p01m = nc.variables['p01m'] totals += np.sum(p01m[:ets_tidx, :, :], axis=0) else: totals = np.sum(p01m[sts_tidx:ets_tidx, :, :], axis=0) nc.close() if np.ma.max(totals) > 0: pass else: print('No StageIV data found, aborting...') sys.exit() # set a small non-zero number to keep things non-zero totals = np.where(totals > 0.001, totals, 0.001) xaxis = np.arange(MYWEST, MYEAST, 0.01) yaxis = np.arange(MYSOUTH, MYNORTH, 0.01) xi, yi = np.meshgrid(xaxis, yaxis) nn = NearestNDInterpolator((lons.flatten(), lats.flatten()), totals.flatten()) STAGE4[:] = nn(xi, yi) write_grid(STAGE4, 'stage4') printt("load_stage4() finished...")
def merge(ts): """ Process an hour's worth of stage4 data into the hourly RE """ # Load up the 12z 24h total, this is what we base our deltas on fn = ("/mesonet/ARCHIVE/data/%s/stage4/ST4.%s.24h.grib") % ( ts.strftime("%Y/%m/%d"), ts.strftime("%Y%m%d%H"), ) if not os.path.isfile(fn): LOG.info("stage4_12z_adjust %s is missing", fn) return False grbs = pygrib.open(fn) grb = grbs[1] val = grb.values lats, lons = grb.latlons() # can save a bit of memory as we don't need all data stride = slice(None, None, 3) lats = np.ravel(lats[stride, stride]) lons = np.ravel(lons[stride, stride]) vals = np.ravel(val[stride, stride]) # Clip large values vals = np.where(vals > 250.0, 0, vals) nn = NearestNDInterpolator((lons, lats), vals) xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS) stage4 = nn(xi, yi) # Prevent Large numbers, negative numbers stage4 = np.where(stage4 < 10000.0, stage4, 0.0) stage4 = np.where(stage4 < 0.0, 0.0, stage4) ts0 = ts - datetime.timedelta(days=1) offset0 = iemre.hourly_offset(ts0) offset1 = iemre.hourly_offset(ts) # Running at 12 UTC 1 Jan if offset0 > offset1: offset0 = 0 # Open up our RE file with ncopen(iemre.get_hourly_ncname(ts.year), "a", timeout=300) as nc: iemre_total = np.sum(nc.variables["p01m"][offset0:offset1, :, :], axis=0) iemre_total = np.where(iemre_total > 0.0, iemre_total, 0.00024) iemre_total = np.where(iemre_total < 10000.0, iemre_total, 0.00024) multiplier = stage4 / iemre_total for offset in range(offset0, offset1): # Get the unmasked dadta data = nc.variables["p01m"][offset, :, :] # Keep data within reason data = np.where(data > 10000.0, 0.0, data) # 0.00024 / 24 adjust = np.where(data > 0, data, 0.00001) * multiplier adjust = np.where(adjust > 250.0, 0, adjust) nc.variables["p01m"][offset, :, :] = np.where( adjust < 0.01, 0, adjust)
def load_stage4(): """ It sucks, but we need to load the stage IV data to give us something to benchmark the MRMS data against, to account for two things: 1) Wind Farms 2) Over-estimates """ printt("load_stage4() called...") # The stage4 files store precip in the rears, so compute 1 AM one_am = datetime.datetime(VALID.year, VALID.month, VALID.day, 12, 0) one_am = one_am.replace(tzinfo=pytz.timezone("UTC")) one_am = one_am.astimezone(pytz.timezone("America/Chicago")) # stage4 data is stored in the rears, so 1am one_am = one_am.replace(hour=1, minute=0, second=0) # clever hack for CST/CDT tomorrow = one_am + datetime.timedelta(hours=36) tomorrow = tomorrow.replace(hour=1) sts_tidx = iemre.hourly_offset(one_am) ets_tidx = iemre.hourly_offset(tomorrow) printt(("stage4 sts_tidx:%s[%s] ets_tidx:%s[%s]") % (sts_tidx, one_am, ets_tidx, tomorrow)) nc = ncopen("/mesonet/data/stage4/%s_stage4_hourly.nc" % (VALID.year, )) p01m = nc.variables["p01m"] lats = nc.variables["lat"][:] lons = nc.variables["lon"][:] # crossing jan 1 if ets_tidx < sts_tidx: printt("Exercise special stageIV logic for jan1!") totals = np.sum(p01m[sts_tidx:, :, :], axis=0) nc.close() nc = ncopen("/mesonet/data/stage4/%s_stage4_hourly.nc" % (tomorrow.year, )) p01m = nc.variables["p01m"] totals += np.sum(p01m[:ets_tidx, :, :], axis=0) else: totals = np.sum(p01m[sts_tidx:ets_tidx, :, :], axis=0) nc.close() if np.ma.max(totals) > 0: pass else: print("No StageIV data found, aborting...") sys.exit() # set a small non-zero number to keep things non-zero totals = np.where(totals > 0.001, totals, 0.001) xaxis = np.arange(MYWEST, MYEAST, 0.01) yaxis = np.arange(MYSOUTH, MYNORTH, 0.01) xi, yi = np.meshgrid(xaxis, yaxis) nn = NearestNDInterpolator((lons.flatten(), lats.flatten()), totals.flatten()) STAGE4[:] = nn(xi, yi) write_grid(STAGE4, "stage4") printt("load_stage4() finished...")
def test_hourly_offset(self): """ Compute the offsets """ ts = datetime.datetime(2013, 1, 1, 0, 0) ts = ts.replace(tzinfo=pytz.timezone("UTC")) offset = iemre.hourly_offset(ts) self.assertEqual(offset, 0) ts = datetime.datetime(2013, 1, 5, 12, 0) ts = ts.replace(tzinfo=pytz.timezone("UTC")) offset = iemre.hourly_offset(ts) self.assertEqual(offset, 4*24 + 12)
def do_precip(nc, ts): """Compute the precip totals based on the hourly analysis totals""" offset = iemre.daily_offset(ts) ets = ts.replace(hour=12, tzinfo=pytz.timezone("UTC")) sts = ets - datetime.timedelta(hours=24) offset1 = iemre.hourly_offset(sts) offset2 = iemre.hourly_offset(ets) hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (ts.year,)) phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0) nc.variables['p01d'][offset] = phour hnc.close()
def merge(ts): """ Process an hour's worth of stage4 data into the hourly RE """ # Load up the 12z 24h total, this is what we base our deltas on fn = ("/mesonet/ARCHIVE/data/%s/stage4/ST4.%s.24h.grib" ) % (ts.strftime("%Y/%m/%d"), ts.strftime("%Y%m%d%H")) grbs = pygrib.open(fn) grb = grbs[1] val = grb.values lats, lons = grb.latlons() # Rough subsample, since the whole enchillata is too much lats = np.ravel(lats[200:-100:5, 300:900:5]) lons = np.ravel(lons[200:-100:5, 300:900:5]) vals = np.ravel(val[200:-100:5, 300:900:5]) # Clip large values vals = np.where(vals > 250., 0, vals) nn = NearestNDInterpolator((lons, lats), vals) xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS) stage4 = nn(xi, yi) # Prevent Large numbers, negative numbers stage4 = np.where(stage4 < 10000., stage4, 0.) stage4 = np.where(stage4 < 0., 0., stage4) # Open up our RE file nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (ts.year,), 'a') ts0 = ts - datetime.timedelta(days=1) offset0 = iemre.hourly_offset(ts0) offset1 = iemre.hourly_offset(ts) # Running at 12 UTC 1 Jan if offset0 > offset1: offset0 = 0 iemre_total = np.sum(nc.variables["p01m"][offset0:offset1, :, :], axis=0) iemre_total = np.where(iemre_total > 0., iemre_total, 0.00024) iemre_total = np.where(iemre_total < 10000., iemre_total, 0.00024) multiplier = stage4 / iemre_total for offset in range(offset0, offset1): # Get the unmasked dadta data = nc.variables["p01m"][offset, :, :] # Keep data within reason data = np.where(data > 10000., 0., data) # 0.00024 / 24 adjust = np.where(data > 0, data, 0.00001) * multiplier adjust = np.where(adjust > 250.0, 0, adjust) nc.variables["p01m"][offset, :, :] = np.where(adjust < 0.01, 0, adjust) ts = ts0 + datetime.timedelta(hours=offset-offset0) nc.sync() nc.close()
def test_hourly_offset(): """ Compute the offsets """ ts = utc(2013, 1, 1, 0, 0) offset = iemre.hourly_offset(ts) assert offset == 0 ts = utc(2013, 1, 1, 6, 0) ts = ts.astimezone(pytz.timezone("America/Chicago")) offset = iemre.hourly_offset(ts) assert offset == 6 ts = utc(2013, 1, 5, 12, 0) offset = iemre.hourly_offset(ts) assert offset == 4*24 + 12
def test_hourly_offset(): """ Compute the offsets """ ts = utc(2013, 1, 1, 0, 0) offset = iemre.hourly_offset(ts) assert offset == 0 ts = utc(2013, 1, 1, 6, 0) ts = ts.astimezone(pytz.timezone("America/Chicago")) offset = iemre.hourly_offset(ts) assert offset == 6 ts = utc(2013, 1, 5, 12, 0) offset = iemre.hourly_offset(ts) assert offset == 4 * 24 + 12
def do_precip12(nc, ts): """Compute the 24 Hour precip at 12 UTC, we do some more tricks though""" offset = iemre.daily_offset(ts) ets = datetime.datetime(ts.year, ts.month, ts.day, 12) ets = ets.replace(tzinfo=pytz.timezone("UTC")) sts = ets - datetime.timedelta(hours=24) offset1 = iemre.hourly_offset(sts) offset2 = iemre.hourly_offset(ets) print(("p01d_12z for %s [idx:%s] %s(%s)->%s(%s)" ) % (ts, offset, sts.strftime("%Y%m%d%H"), offset1, ets.strftime("%Y%m%d%H"), offset2)) hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (ts.year,)) phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0) nc.variables['p01d_12z'][offset] = phour hnc.close()
def merge(valid): """ Process an hour's worth of stage4 data into the hourly RE """ nc = netCDF4.Dataset( ("/mesonet/data/stage4/%s_stage4_hourly.nc") % (valid.year, ), 'r') tidx = iemre.hourly_offset(valid) val = nc.variables['p01m'][tidx, :, :] # print("stage4 mean: %.2f max: %.2f" % (np.mean(val), np.max(val))) lats = nc.variables['lat'][:] lons = nc.variables['lon'][:] # Rough subsample, since the whole enchillata is too much lats = np.ravel(lats[200:-100:5, 300:900:5]) lons = np.ravel(lons[200:-100:5, 300:900:5]) vals = np.ravel(val[200:-100:5, 300:900:5]) nn = NearestNDInterpolator((lons, lats), vals) xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS) res = nn(xi, yi) # Lets clip bad data # 10 inches per hour is bad data res = np.where(np.logical_or(res < 0, res > 250), 0., res) # print("Resulting mean: %.2f max: %.2f" % (np.mean(res), np.max(res))) # Open up our RE file nc = netCDF4.Dataset( "/mesonet/data/iemre/%s_mw_hourly.nc" % (valid.year, ), 'a') nc.variables["p01m"][tidx, :, :] = res # print(("Readback mean: %.2f max: %.2f" # ) % (np.mean(nc.variables["p01m"][tidx, :, :]), # np.max(nc.variables["p01m"][tidx, :, :]))) nc.close()
def to_netcdf(valid): """Persist this 1 hour precip information to the netcdf storage Recall that this timestep has data for the previous hour""" fn = ("/mesonet/ARCHIVE/data/%s/stage4/ST4.%s.01h.grib") % ( valid.strftime("%Y/%m/%d"), valid.strftime("%Y%m%d%H")) if not os.path.isfile(fn): print("stage4_hourlyre: missing stageIV hourly file %s" % (fn, )) return False gribs = pygrib.open(fn) grb = gribs[1] val = grb.values # values over 10 inches are bad val = np.where(val > 250., 0, val) nc = netCDF4.Dataset( ("/mesonet/data/stage4/%s_stage4_hourly.nc") % (valid.year, ), 'a') tidx = iemre.hourly_offset(valid) if nc.variables['p01m_status'][tidx] > 1: print("Skipping stage4_hourlyre write as variable status is >1") nc.close() return True p01m = nc.variables['p01m'] p01m[tidx, :, :] = val nc.variables['p01m_status'][tidx] = 1 nc.close() return True
def compute_stage4(lon, lat, year): """Build a daily dataframe for the stage4 data""" nc = netCDF4.Dataset("/mesonet/data/stage4/%s_stage4_hourly.nc" % (year,)) lons = nc.variables['lon'][:] lats = nc.variables['lat'][:] dist = ((lons - lon)**2 + (lats - lat)**2)**0.5 (yidx, xidx) = np.unravel_index(dist.argmin(), dist.shape) print(("Computed stage4 nclon:%.2f nclat:%.2f yidx:%s xidx:%s " ) % (lons[yidx, xidx], lats[yidx, xidx], yidx, xidx)) p01i = distance(nc.variables['p01m'][:, yidx, xidx], 'MM').value('IN') nc.close() df = pd.DataFrame({'precip': 0.0}, index=pd.date_range('%s-01-01' % (year, ), '%s-12-31' % (year, ), tz='America/Chicago')) for date in df.index.values: date2 = datetime.datetime.utcfromtimestamp(date.tolist()/1e9) ts = datetime.datetime(date2.year, date2.month, date2.day, 6) ts = ts.replace(tzinfo=pytz.utc) ts = ts.astimezone(pytz.timezone("America/Chicago")) ts = ts.replace(hour=0) ts = ts.astimezone(pytz.utc) tidx = hourly_offset(ts) # values are in the rears val = np.ma.sum(p01i[tidx+1:tidx+25]) if val > 0: df.at[date, 'precip'] = val # close enough return df
def to_netcdf(valid): """Persist this 1 hour precip information to the netcdf storage Recall that this timestep has data for the previous hour""" fn = ("/mesonet/ARCHIVE/data/%s/model/NARR/apcp_%s.grib") % ( valid.strftime("%Y/%m/%d"), valid.strftime("%Y%m%d%H%M"), ) if not os.path.isfile(fn): print("merge_narr: missing file %s" % (fn,)) return False gribs = pygrib.open(fn) grb = gribs[1] val = grb.values nc = ncopen( ("/mesonet/data/iemre/%s_narr.nc") % (valid.year,), "a", timeout=300 ) tidx = int((iemre.hourly_offset(valid) + 1) / 3) print("%s np.min: %s np.max: %s" % (tidx, np.min(val), np.max(val))) apcp = nc.variables["apcp"] apcp[tidx, :, :] = val nc.close() return True
def merge(valid): """ Process an hour's worth of stage4 data into the hourly RE """ nc = ncopen(("/mesonet/data/stage4/%s_stage4_hourly.nc" ) % (valid.year, ), 'r') tidx = iemre.hourly_offset(valid) val = nc.variables['p01m'][tidx, :, :] # print("stage4 mean: %.2f max: %.2f" % (np.mean(val), np.max(val))) lats = nc.variables['lat'][:] lons = nc.variables['lon'][:] # Our data is 4km, iemre is 0.125deg, so we stride some to cut down on mem stride = slice(None, None, 3) lats = np.ravel(lats[stride, stride]) lons = np.ravel(lons[stride, stride]) vals = np.ravel(val[stride, stride]) nn = NearestNDInterpolator((lons, lats), vals) xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS) res = nn(xi, yi) # Lets clip bad data # 10 inches per hour is bad data res = np.where(np.logical_or(res < 0, res > 250), 0., res) # print("Resulting mean: %.2f max: %.2f" % (np.mean(res), np.max(res))) # Open up our RE file nc = ncopen(iemre.get_hourly_ncname(valid.year), 'a', timeout=300) nc.variables["p01m"][tidx, :, :] = res # print(("Readback mean: %.2f max: %.2f" # ) % (np.mean(nc.variables["p01m"][tidx, :, :]), # np.max(nc.variables["p01m"][tidx, :, :]))) nc.close()
def compute_stage4(lon, lat, year): """Build a daily dataframe for the stage4 data""" nc = netCDF4.Dataset("/mesonet/data/stage4/%s_stage4_hourly.nc" % (year, )) lons = nc.variables['lon'][:] lats = nc.variables['lat'][:] dist = ((lons - lon)**2 + (lats - lat)**2)**0.5 (yidx, xidx) = np.unravel_index(dist.argmin(), dist.shape) print(("Computed stage4 nclon:%.2f nclat:%.2f yidx:%s xidx:%s ") % (lons[yidx, xidx], lats[yidx, xidx], yidx, xidx)) p01i = distance(nc.variables['p01m'][:, yidx, xidx], 'MM').value('IN') nc.close() df = pd.DataFrame({'precip': 0.0}, index=pd.date_range('%s-01-01' % (year, ), '%s-12-31' % (year, ), tz='America/Chicago')) for date in df.index.values: date2 = datetime.datetime.utcfromtimestamp(date.tolist() / 1e9) ts = datetime.datetime(date2.year, date2.month, date2.day, 6) ts = ts.replace(tzinfo=pytz.utc) ts = ts.astimezone(pytz.timezone("America/Chicago")) ts = ts.replace(hour=0) ts = ts.astimezone(pytz.utc) tidx = hourly_offset(ts) # values are in the rears val = np.ma.sum(p01i[tidx + 1:tidx + 25]) if val > 0: df.at[date, 'precip'] = val # close enough return df
def copy_to_iemre(valid): """verbatim copy over to IEMRE.""" tidx = iemre.hourly_offset(valid) nc = ncopen(("/mesonet/data/stage4/%s_stage4_hourly.nc") % (valid.year, ), 'a', timeout=300) lats = nc.variables['lat'][:] lons = nc.variables['lon'][:] val = nc.variables['p01m'][tidx] nc.close() # Our data is 4km, iemre is 0.125deg, so we stride some to cut down on mem stride = slice(None, None, 3) lats = np.ravel(lats[stride, stride]) lons = np.ravel(lons[stride, stride]) vals = np.ravel(val[stride, stride]) nn = NearestNDInterpolator((lons, lats), vals) xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS) res = nn(xi, yi) # Lets clip bad data # 10 inches per hour is bad data res = np.where(np.logical_or(res < 0, res > 250), 0., res) # Open up our RE file nc = ncopen(iemre.get_hourly_ncname(valid.year), 'a', timeout=300) nc.variables["p01m"][tidx, :, :] = res LOG.debug("wrote data to hourly IEMRE min: %.2f avg: %.2f max: %.2f", np.min(res), np.mean(res), np.max(res)) nc.close()
def ingest_hourly_grib(valid): """Copy the hourly grib data into the netcdf storage. Returns: int value of the new p01m_status """ tidx = iemre.hourly_offset(valid) fn = valid.strftime( "/mesonet/ARCHIVE/data/%Y/%m/%d/stage4/ST4.%Y%m%d%H.01h.grib") if not os.path.isfile(fn): LOG.info("stage4_ingest: missing file %s", fn) return 0 gribs = pygrib.open(fn) grb = gribs[1] val = grb.values # values over 10 inches are bad val = np.where(val > 250., 0, val) nc = ncopen(("/mesonet/data/stage4/%s_stage4_hourly.nc") % (valid.year, ), 'a', timeout=300) p01m = nc.variables['p01m'] # account for legacy grid prior to 2002 if val.shape == (880, 1160): p01m[tidx, 1:, :] = val[:, 39:] else: p01m[tidx, :, :] = val nc.variables['p01m_status'][tidx] = 1 LOG.debug("write p01m to stage4 netcdf min: %.2f avg: %.2f max: %.2f", np.min(val), np.mean(val), np.max(val)) nc.close() return 1
def test_hourly_offset(self): """ Compute the offsets """ ts = datetime.datetime(2013, 1, 1, 0, 0) ts = ts.replace(tzinfo=pytz.timezone("UTC")) offset = iemre.hourly_offset(ts) self.assertEqual(offset, 0) ts = datetime.datetime(2013, 1, 1, 6, 0) ts = ts.replace(tzinfo=pytz.timezone("UTC")) ts = ts.astimezone(pytz.timezone("America/Chicago")) offset = iemre.hourly_offset(ts) self.assertEqual(offset, 6) ts = datetime.datetime(2013, 1, 5, 12, 0) ts = ts.replace(tzinfo=pytz.timezone("UTC")) offset = iemre.hourly_offset(ts) self.assertEqual(offset, 4 * 24 + 12)
def write_grid(valid, vname, grid): """Atomic write of data to our netcdf storage This is isolated so that we don't 'lock' up our file while intensive work is done """ offset = iemre.hourly_offset(valid) with ncopen(iemre.get_hourly_ncname(valid.year), "a", timeout=300) as nc: nc.variables[vname][offset] = grid
def do_precip(ts, ds): """Compute the 6 UTC to 6 UTC precip We need to be careful here as the timestamp sent to this app is today, we are actually creating the analysis for yesterday """ sts = utc(ts.year, ts.month, ts.day, 6) ets = sts + datetime.timedelta(hours=24) offset = iemre.daily_offset(ts) offset1 = iemre.hourly_offset(sts) offset2 = iemre.hourly_offset(ets) if ts.month == 12 and ts.day == 31: LOG.warning( "p01d for %s [idx:%s] %s(%s)->%s(%s) SPECIAL", ts, offset, sts.strftime("%Y%m%d%H"), offset1, ets.strftime("%Y%m%d%H"), offset2, ) ncfn = iemre.get_hourly_ncname(ets.year) if not os.path.isfile(ncfn): LOG.warning("Missing %s", ncfn) return with ncopen(ncfn, timeout=600) as hnc: phour = np.sum(hnc.variables["p01m"][:offset2, :, :], 0) ncfn = iemre.get_hourly_ncname(sts.year) if os.path.isfile(ncfn): with ncopen(ncfn, timeout=600) as hnc: phour += np.sum(hnc.variables["p01m"][offset1:, :, :], 0) else: ncfn = iemre.get_hourly_ncname(sts.year) if not os.path.isfile(ncfn): LOG.warning("Missing %s", ncfn) return with ncopen(ncfn, timeout=600) as hnc: # for offset in range(offset1, offset2): # LOG.info( # "offset: %s min: %s max: %s", # offset, np.ma.min(hnc.variables['p01m'][offset, :, :]), # np.max(hnc.variables['p01m'][offset, :, :])) phour = np.sum(hnc.variables["p01m"][offset1:offset2, :, :], 0) ds["p01d"].values = np.where(phour < 0, 0, phour)
def get_p01m_status(valid): """Figure out what our current status is of this hour.""" nc = ncopen(("/mesonet/data/stage4/%s_stage4_hourly.nc") % (valid.year, ), timeout=300) tidx = iemre.hourly_offset(valid) # 2 prism_adjust_stage4 ran # 1 copied hourly data in # 0 nothing happened p01m_status = nc.variables['p01m_status'][tidx] nc.close() LOG.debug("p01m_status is %s for valid %s", p01m_status, valid) return p01m_status
def do_precip(nc, ts): """Compute the 6 UTC to 6 UTC precip We need to be careful here as the timestamp sent to this app is today, we are actually creating the analysis for yesterday """ sts = utc(ts.year, ts.month, ts.day, 6) ets = sts + datetime.timedelta(hours=24) offset = iemre.daily_offset(ts) offset1 = iemre.hourly_offset(sts) offset2 = iemre.hourly_offset(ets) if ts.month == 12 and ts.day == 31: print(("p01d for %s [idx:%s] %s(%s)->%s(%s) SPECIAL" ) % (ts, offset, sts.strftime("%Y%m%d%H"), offset1, ets.strftime("%Y%m%d%H"), offset2)) ncfn = "/mesonet/data/iemre/%s_mw_hourly.nc" % (ets.year,) if not os.path.isfile(ncfn): print("Missing %s" % (ncfn,)) return hnc = netCDF4.Dataset(ncfn) phour = np.sum(hnc.variables['p01m'][:offset2, :, :], 0) hnc.close() hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % ( sts.year,)) phour += np.sum(hnc.variables['p01m'][offset1:, :, :], 0) hnc.close() else: if sts.year >= 1900: print(("p01d for %s [idx:%s] %s(%s)->%s(%s)" ) % (ts, offset, sts.strftime("%Y%m%d%H"), offset1, ets.strftime("%Y%m%d%H"), offset2)) ncfn = "/mesonet/data/iemre/%s_mw_hourly.nc" % (sts.year,) if not os.path.isfile(ncfn): print("Missing %s" % (ncfn,)) return hnc = netCDF4.Dataset(ncfn) phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0) hnc.close() nc.variables['p01d'][offset] = np.where(phour < 0, 0, phour)
def main(): """Go Main Go""" nc = netCDF4.Dataset('/mesonet/data/stage4/2018_stage4_hourly.nc') precip = nc.variables['p01m'] # Compute needed grid bounds # y, x # NW 678, 412 # SW 313, 417 # NE 764, 787 # SE 432, 941 # (west, east, south, north) = compute_bounds(nc) south = 313 north = 678 west = 412 east = 900 lats = nc.variables['lat'][south:north, west:east] lons = nc.variables['lon'][south:north, west:east] pts = [] for lon, lat in zip(np.ravel(lons), np.ravel(lats)): pts.append(Point(lon, lat)) df = gpd.GeoDataFrame({'geometry': pts}) # iterate over days now = datetime.date(2018, 7, 1) ets = datetime.date(2019, 1, 1) while now < ets: valid = util.utc(now.year, now.month, now.day, 5) tidx0 = iemre.hourly_offset(valid) now += datetime.timedelta(days=1) valid = util.utc(now.year, now.month, now.day, 5) tidx1 = iemre.hourly_offset(valid) data = precip[tidx0:tidx1, south:north, west:east] total = np.sum(data, axis=0) print("Processing %s max precip: %.2f" % ( now - datetime.timedelta(days=1), np.max(total))) df[(now - datetime.timedelta(days=1)).strftime("%b%d") ] = distance(np.ravel(total), 'MM').value("IN") df.to_file('combined.shp')
def main(): """Go Main Go""" sts = datetime.datetime(2018, 4, 20, 0) sts = sts.replace(tzinfo=pytz.utc) ets = datetime.datetime(2018, 5, 11, 0) ets = ets.replace(tzinfo=pytz.utc) nc = ncopen(iemre.get_hourly_ncname(sts.year)) lons = nc.variables['lon'][:] lats = nc.variables['lat'][:] running = np.zeros((len(nc.dimensions['lat']), len(nc.dimensions['lon']))) maxval = np.zeros((len(nc.dimensions['lat']), len(nc.dimensions['lon']))) interval = datetime.timedelta(hours=1) now = sts i, j = iemre.find_ij(-93.61, 41.99) while now < ets: offset = iemre.hourly_offset(now) p01m = np.sum(nc.variables['p01m'][offset - 24:offset], axis=0) # 0.05in is 1.27 mm this = np.where(p01m > THRESHOLD, 1, 0) running = np.where(this == 1, 0, running + 1) maxval = np.where(running > maxval, running, maxval) print("%s %s %s" % (now, running[j, i], maxval[j, i])) now += interval # maxval = numpy.where(domain == 1, maxval, 1.e20) m = plot.MapPlot(sector='midwest', title=('Max Period ' 'between 24 Hour 0.25+ inch Total Precipitation'), subtitle=('Period of 20 Apr - 11 May 2018, ' 'based on NCEP Stage IV data')) extra = lons[-1] + (lons[-1] - lons[-2]) lons[-1] = extra # lons = np.concatenate([lons, [extra, ]]) extra = lats[-1] + (lats[-1] - lats[-2]) lats[-1] = extra # lats = np.concatenate([lats, [extra, ]]) lons, lats = np.meshgrid(lons, lats) # m.pcolormesh(x, y, maxval / 24.0, numpy.arange(0,25,1), units='days') maxval = np.where(maxval > 800, 73., maxval) cmap = plt.get_cmap('terrain') m.contourf(lons, lats, maxval / 24.0, np.arange(1, 11.1, 1), cmap=cmap, units='days', clip_on=False) m.postprocess(filename='test.png')
def dowork(fields): """Do work!""" date = datetime.datetime.strptime(fields.get("valid"), "%Y-%m-%d") lat = float(fields.get("lat")) lon = float(fields.get("lon")) # We want data for the UTC date and timestamps are in the rears, so from # 1z through 1z sts = utc(date.year, date.month, date.day, 1) ets = sts + datetime.timedelta(hours=24) sidx = iemre.hourly_offset(sts) eidx = iemre.hourly_offset(ets) ncfn = "/mesonet/data/stage4/%s_stage4_hourly.nc" % (date.year, ) res = {"gridi": -1, "gridj": -1, "data": []} if not os.path.isfile(ncfn): return json.dumps(res) with ncopen(ncfn) as nc: dist = ((nc.variables["lon"][:] - lon)**2 + (nc.variables["lat"][:] - lat)**2)**0.5 (j, i) = np.unravel_index(dist.argmin(), dist.shape) res["gridi"] = int(i) res["gridj"] = int(j) ppt = nc.variables["p01m"][sidx:eidx, j, i] for tx, pt in enumerate(ppt): valid = sts + datetime.timedelta(hours=tx) res["data"].append({ "end_valid": valid.strftime("%Y-%m-%dT%H:00:00Z"), "precip_in": myrounder(datatypes.distance(pt, "MM").value("IN"), 2), }) return json.dumps(res)
def do_precip(ts): """Compute the 6 UTC to 6 UTC precip We need to be careful here as the timestamp sent to this app is today, we are actually creating the analysis for yesterday """ sts = utc(ts.year, ts.month, ts.day, 6) ets = sts + datetime.timedelta(hours=24) offset = iemre.daily_offset(ts) offset1 = iemre.hourly_offset(sts) offset2 = iemre.hourly_offset(ets) if ts.month == 12 and ts.day == 31: print(("p01d for %s [idx:%s] %s(%s)->%s(%s) SPECIAL") % (ts, offset, sts.strftime("%Y%m%d%H"), offset1, ets.strftime("%Y%m%d%H"), offset2)) ncfn = iemre.get_hourly_ncname(ets.year) if not os.path.isfile(ncfn): print("Missing %s" % (ncfn, )) return hnc = ncopen(ncfn, timeout=600) phour = np.sum(hnc.variables['p01m'][:offset2, :, :], 0) hnc.close() ncfn = iemre.get_hourly_ncname(sts.year) if os.path.isfile(ncfn): hnc = ncopen(ncfn, timeout=600) phour += np.sum(hnc.variables['p01m'][offset1:, :, :], 0) hnc.close() else: ncfn = iemre.get_hourly_ncname(sts.year) if not os.path.isfile(ncfn): print("Missing %s" % (ncfn, )) return hnc = ncopen(ncfn, timeout=600) phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0) hnc.close() write_grid(ts, 'p01d', np.where(phour < 0, 0, phour))
def do_precip12(nc, ts): """Compute the 24 Hour precip at 12 UTC, we do some more tricks though""" offset = iemre.daily_offset(ts) ets = utc(ts.year, ts.month, ts.day, 12) sts = ets - datetime.timedelta(hours=24) offset1 = iemre.hourly_offset(sts) offset2 = iemre.hourly_offset(ets) if ts.month == 1 and ts.day == 1: if sts.year >= 1900: print(("p01d_12z for %s [idx:%s] %s(%s)->%s(%s) SPECIAL") % (ts, offset, sts.strftime("%Y%m%d%H"), offset1, ets.strftime("%Y%m%d%H"), offset2)) ncfn = "/mesonet/data/iemre/%s_mw_hourly.nc" % (ets.year, ) if not os.path.isfile(ncfn): print("Missing %s" % (ncfn, )) return hnc = netCDF4.Dataset(ncfn) phour = np.sum(hnc.variables['p01m'][:offset2, :, :], 0) hnc.close() hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (sts.year, )) phour += np.sum(hnc.variables['p01m'][offset1:, :, :], 0) hnc.close() else: if sts.year >= 1900: print(("p01d_12z for %s [idx:%s] %s(%s)->%s(%s)") % (ts, offset, sts.strftime("%Y%m%d%H"), offset1, ets.strftime("%Y%m%d%H"), offset2)) ncfn = "/mesonet/data/iemre/%s_mw_hourly.nc" % (ts.year, ) if not os.path.isfile(ncfn): print("Missing %s" % (ncfn, )) return hnc = netCDF4.Dataset(ncfn) phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0) hnc.close() nc.variables['p01d_12z'][offset] = phour
def workflow(sts, ets, i, j): """Return a dict of our data.""" res = [] # BUG here for Dec 31. fn = iemre.get_hourly_ncname(sts.year) if not os.path.isfile(fn): return res if i is None or j is None: return {"error": "Coordinates outside of domain"} UTC = ZoneInfo("UTC") with ncopen(fn) as nc: now = sts while now <= ets: offset = iemre.hourly_offset(now) res.append( { "valid_utc": now.astimezone(UTC).strftime(ISO), "valid_local": now.strftime(ISO[:-1]), "skyc_%": myrounder(nc.variables["skyc"][offset, j, i], 1), "air_temp_f": myrounder( convert_value( nc.variables["tmpk"][offset, j, i], "degK", "degF" ), 1, ), "dew_point_f": myrounder( convert_value( nc.variables["dwpk"][offset, j, i], "degK", "degF" ), 1, ), "uwnd_mps": myrounder( nc.variables["uwnd"][offset, j, i], 2 ), "vwnd_mps": myrounder( nc.variables["vwnd"][offset, j, i], 2 ), "hourly_precip_in": myrounder( mm2inch(nc.variables["p01m"][offset, j, i]), 2 ), } ) now += datetime.timedelta(hours=1) return pd.DataFrame(res)
def workflow(sts, ets, i, j): """Return a dict of our data.""" res = {"data": [], "generated_at": utc().strftime(ISO)} # BUG here for Dec 31. fn = iemre.get_hourly_ncname(sts.year) if not os.path.isfile(fn): return res if i is None or j is None: return {"error": "Coordinates outside of domain"} with ncopen(fn) as nc: now = sts while now <= ets: offset = iemre.hourly_offset(now) res["data"].append( { "valid_utc": now.astimezone(pytz.UTC).strftime(ISO), "valid_local": now.strftime(ISO), "skyc_%": myrounder(nc.variables["skyc"][offset, j, i], 1), "air_temp_f": myrounder( datatypes.temperature( nc.variables["tmpk"][offset, j, i], "K" ).value("F"), 1, ), "dew_point_f": myrounder( datatypes.temperature( nc.variables["dwpk"][offset, j, i], "K" ).value("F"), 1, ), "uwnd_mps": myrounder( nc.variables["uwnd"][offset, j, i], 2 ), "vwnd_mps": myrounder( nc.variables["vwnd"][offset, j, i], 2 ), "hourly_precip_in": myrounder( nc.variables["p01m"][offset, j, i] / 25.4, 2 ), } ) now += datetime.timedelta(hours=1) return res
def main(argv): """Go Main Go.""" log = logger() if len(argv) == 6: valid = utc(int(argv[1]), int(argv[2]), int(argv[3]), int(argv[4])) ncfn = iemre.get_hourly_ncname(valid.year) idx = iemre.hourly_offset(valid) else: valid = datetime.date(int(argv[1]), int(argv[2]), int(argv[3])) ncfn = iemre.get_daily_ncname(valid.year) idx = iemre.daily_offset(valid) ds = iemre.get_grids(valid) with ncopen(ncfn, 'a', timeout=600) as nc: for vname in ds: if vname not in nc.variables: continue log.debug("copying database var %s to netcdf", vname) nc.variables[vname][idx, :, :] = ds[vname].values
def merge(ts): """ Process an hour's worth of stage4 data into the hourly RE """ fn = ("/mesonet/ARCHIVE/data/%s/stage4/ST4.%s.01h.grib" ) % (ts.strftime("%Y/%m/%d"), ts.strftime("%Y%m%d%H")) if os.path.isfile(fn): gribs = pygrib.open(fn) grb = gribs[1] val = grb.values lats, lons = grb.latlons() # Rough subsample, since the whole enchillata is too much lats = np.ravel(lats[200:-100:5, 300:900:5]) lons = np.ravel(lons[200:-100:5, 300:900:5]) vals = np.ravel(val[200:-100:5, 300:900:5]) # Clip large values vals = np.where(vals > 250., 0, vals) nn = NearestNDInterpolator((lons, lats), vals) xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS) res = nn(xi, yi) else: print 'Missing stage4 %s' % (fn,) res = np.zeros((iemre.NX, iemre.NY)) # Lets clip bad data res = np.where(res < 0, 0., res) # 10 inches per hour is bad data res = np.where(res > 250., 0., res) # Open up our RE file nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (ts.year,), 'a') offset = iemre.hourly_offset(ts) nc.variables["p01m"][offset, :, :] = res nc.close() del nc
def grid_hour(nc, ts): """ I proctor the gridding of data on an hourly basis @param ts Timestamp of the analysis, we'll consider a 20 minute window """ ts0 = ts - datetime.timedelta(minutes=10) ts1 = ts + datetime.timedelta(minutes=10) offset = iemre.hourly_offset(ts) utcnow = datetime.datetime.utcnow() utcnow = utcnow.replace(tzinfo=pytz.timezone("UTC")) - datetime.timedelta(hours=36) # If we are near realtime, look in IEMAccess instead of ASOS database if utcnow < ts: dbconn = psycopg2.connect(database="iem", host="iemdb", user="******") pcursor = dbconn.cursor(cursor_factory=psycopg2.extras.DictCursor) table = "current_log" pcolumn = "(phour * 25.4)" sql = """SELECT t.id as station, max(case when tmpf > -60 and tmpf < 130 THEN tmpf else null end) as max_tmpf, max(case when sknt > 0 and sknt < 100 then sknt else 0 end) as max_sknt, max(getskyc(skyc1)) as max_skyc1, max(getskyc(skyc2)) as max_skyc2, max(getskyc(skyc3)) as max_skyc3, max(case when %s > 0 and %s < 1000 then %s else 0 end) as max_p01m, max(case when dwpf > -60 and dwpf < 100 THEN dwpf else null end) as max_dwpf, max(case when sknt >= 0 then sknt else 0 end) as sknt, max(case when sknt >= 0 then drct else 0 end) as drct from %s s, stations t WHERE t.id in %s and t.iemid = s.iemid and valid >= '%s' and valid < '%s' GROUP by station """ % ( pcolumn, pcolumn, pcolumn, table, ids, ts0, ts1, ) else: dbconn = psycopg2.connect(database="asos", host="iemdb", user="******") pcursor = dbconn.cursor(cursor_factory=psycopg2.extras.DictCursor) table = "t%s" % (ts.year,) pcolumn = "p01i" sql = """SELECT station, max(case when tmpf > -60 and tmpf < 130 THEN tmpf else null end) as max_tmpf, max(case when sknt > 0 and sknt < 100 then sknt else 0 end) as max_sknt, max(getskyc(skyc1)) as max_skyc1, max(getskyc(skyc2)) as max_skyc2, max(getskyc(skyc3)) as max_skyc3, max(case when %s > 0 and %s < 1000 then %s else 0 end) as max_p01m, max(case when dwpf > -60 and dwpf < 100 THEN dwpf else null end) as max_dwpf, max(case when sknt >= 0 then sknt else 0 end) as sknt, max(case when sknt >= 0 then drct else 0 end) as drct from %s WHERE station in %s and valid >= '%s' and valid < '%s' GROUP by station """ % ( pcolumn, pcolumn, pcolumn, table, ids, ts0, ts1, ) pcursor.execute(sql) if pcursor.rowcount > 4: rs = [] for row in pcursor: rs.append(row) ures, vres = grid_wind(rs) if ures is not None: nc.variables["uwnd"][offset] = ures nc.variables["vwnd"][offset] = vres res = generic_gridder(rs, "max_tmpf") if res is not None: nc.variables["tmpk"][offset] = dt.temperature(res, "F").value("K") res = grid_skyc(rs) if res is not None: nc.variables["skyc"][offset] = res else: print "%s has %02i entries, FAIL" % (ts.strftime("%Y-%m-%d %H:%M"), pcursor.rowcount)
sts = datetime.datetime(2013,5,1, 0) sts = sts.replace(tzinfo=pytz.timezone("UTC")) ets = datetime.datetime(2013,6,10, 0) ets = ets.replace(tzinfo=pytz.timezone("UTC")) nc = netCDF4.Dataset('/mesonet/data/iemre/2013_mw_hourly.nc') lons = nc.variables['lon'][:] lats = nc.variables['lat'][:] running = numpy.zeros( (len(nc.dimensions['lat']), len(nc.dimensions['lon']))) maxval = numpy.zeros( (len(nc.dimensions['lat']), len(nc.dimensions['lon']))) interval = datetime.timedelta(hours=1) now = sts i,j = iemre.find_ij(-93.61, 41.99) while now < ets: offset = iemre.hourly_offset(now) p01m = nc.variables['p01m'][offset] # 0.05in is 1.27 mm this = numpy.where(p01m > 1.27, 1, 0) running = numpy.where(this == 1, 0, running + 1) maxval = numpy.where(running > maxval, running, maxval) print now, running[j,i], maxval[j,i] now += interval nc2 = netCDF4.Dataset("/mesonet/data/iemre/state_weights.nc") domain = nc2.variables['domain'][:] nc2.close() maxval = numpy.where(domain == 1, maxval, 1.e20) m = plot.MapPlot(sector='midwest',