def estimate_precip(df, ts):
"""Estimate precipitation based on IEMRE"""
idx = iemre.daily_offset(ts)
nc = ncopen(iemre.get_daily_ncname(ts.year), 'r', timeout=300)
grid12 = distance(nc.variables['p01d_12z'][idx, :, :],
'MM').value("IN").filled(0)
grid00 = distance(nc.variables['p01d'][idx, :, :],
"MM").value("IN").filled(0)
nc.close()
for sid, row in df.iterrows():
if not pd.isnull(row['precip']):
continue
if row['precip24_hour'] in [0, 22, 23]:
precip = grid00[row['gridj'], row['gridi']]
else:
precip = grid12[row['gridj'], row['gridi']]
# denote trace
if precip > 0 and precip < 0.01:
df.at[sid, 'precip'] = TRACE_VALUE
elif precip < 0:
df.at[sid, 'precip'] = 0
elif np.isnan(precip) or np.ma.is_masked(precip):
df.at[sid, 'precip'] = 0
else:
df.at[sid, 'precip'] = "%.2f" % (precip,)
def replace_obs_iem(df, location):
"""Replace dataframe data with obs for this location
Tricky part, if the baseline already provides data for this year, we should
use it!
"""
pgconn = get_dbconn("iem", user="******")
cursor = pgconn.cursor()
station = XREF[location]["station"]
today = datetime.date.today()
jan1 = today.replace(month=1, day=1)
years = [
int(y) for y in np.arange(df.index.values.min().year,
df.index.values.max().year + 1)
]
table = "summary_%s" % (jan1.year, )
cursor.execute(
"""
select day, max_tmpf, min_tmpf, srad_mj, pday
from """ + table + """ s JOIN stations t on (s.iemid = t.iemid)
WHERE t.id = %s and max_tmpf is not null
and day < 'TODAY' ORDER by day ASC
""",
(station, ),
)
rcols = ["maxt", "mint", "radn", "gdd", "rain"]
replaced = []
for row in cursor:
valid = row[0]
# Does our df currently have data for this date? If so, we shall do
# no more
dont_replace = not np.isnan(df.at[valid, "mint"])
if not dont_replace:
replaced.append(valid)
_gdd = gdd(temperature(row[1], "F"), temperature(row[2], "F"))
for year in years:
if valid.month == 2 and valid.day == 29 and year % 4 != 0:
continue
if dont_replace:
df.loc[valid.replace(year=year), rcols[3:]] = (
_gdd,
distance(row[4], "in").value("mm"),
)
continue
df.loc[valid.replace(year=year), rcols] = (
temperature(row[1], "F").value("C"),
temperature(row[2], "F").value("C"),
row[3],
_gdd,
distance(row[4], "in").value("mm"),
)
if replaced:
LOG.info(
" used IEM Access %s from %s->%s",
station,
replaced[0],
replaced[-1],
)
def update_precip(date, station, hdf):
"""Do the update work"""
sts = datetime.datetime(date.year, date.month, date.day, 7)
sts = sts.replace(tzinfo=pytz.utc)
ets = sts + datetime.timedelta(hours=24)
ldf = hdf[(hdf['station'] == station) & (hdf['valid'] >= sts) &
(hdf['valid'] < ets)]
newpday = distance(ldf['precip_in'].sum(), 'IN')
# update iemaccess
pgconn = get_dbconn('iem')
cursor = pgconn.cursor()
cursor.execute(
"""
UPDATE summary s
SET pday = %s
FROM stations t
WHERE t.iemid = s.iemid and s.day = %s and t.id = %s and
t.network = 'ISUSM'
""", (newpday.value('IN'), date, station))
cursor.close()
pgconn.commit()
# update isusm
pgconn = get_dbconn('isuag')
cursor = pgconn.cursor()
# daily
cursor.execute(
"""
UPDATE sm_daily
SET rain_mm_tot_qc = %s, rain_mm_tot_f = 'E'
WHERE valid = %s and station = %s
""", (newpday.value('MM'), date, station))
for _, row in ldf.iterrows():
# hourly
total = distance(row['precip_in'], 'IN').value('MM')
cursor.execute(
"""
UPDATE sm_hourly
SET rain_mm_tot_qc = %s, rain_mm_tot_f = 'E'
WHERE valid = %s and station = %s
""", (total, row['valid'], station))
# 15minute
for minute in [45, 30, 15, 0]:
cursor.execute(
"""
UPDATE sm_15minute
SET rain_mm_tot_qc = %s, rain_mm_tot_f = 'E'
WHERE valid = %s and station = %s
""", (total / 4.,
row['valid'] - datetime.timedelta(minutes=minute), station))
cursor.close()
pgconn.commit()
def update_precip(date, station, hdf):
"""Do the update work"""
sts = datetime.datetime(date.year, date.month, date.day, 7)
sts = sts.replace(tzinfo=pytz.utc)
ets = sts + datetime.timedelta(hours=24)
ldf = hdf[
(hdf["station"] == station)
& (hdf["valid"] >= sts)
& (hdf["valid"] < ets)
]
newpday = distance(ldf["precip_in"].sum(), "IN")
set_iemacces(station, date, newpday.value("IN"))
# update isusm
pgconn = get_dbconn("isuag")
cursor = pgconn.cursor()
# daily
cursor.execute(
"""
UPDATE sm_daily
SET rain_mm_tot_qc = %s, rain_mm_tot_f = 'E'
WHERE valid = %s and station = %s
""",
(newpday.value("MM"), date, station),
)
for _, row in ldf.iterrows():
# hourly
total = distance(row["precip_in"], "IN").value("MM")
cursor.execute(
"""
UPDATE sm_hourly
SET rain_mm_tot_qc = %s, rain_mm_tot_f = 'E'
WHERE valid = %s and station = %s
""",
(total, row["valid"], station),
)
# minute
for minute in range(60):
cursor.execute(
"""
UPDATE sm_minute
SET rain_in_tot_qc = %s, rain_in_tot_f = 'E'
WHERE valid = %s and station = %s
""",
(
total / 60.0 / 25.4,
row["valid"] - datetime.timedelta(minutes=minute),
station,
),
)
cursor.close()
pgconn.commit()
def varconv(val, element):
"""Convert NCDC to something we use in the database"""
ret = None
if element in ["TMAX", "TMIN"]:
fv = round(temperature(float(val) / 10.0, "C").value("F"), 0)
ret = None if (fv < -100 or fv > 150) else fv
elif element in ["PRCP"]:
fv = round(distance(float(val) / 10.0, "MM").value("IN"), 2)
ret = None if fv < 0 else fv
elif element in ["SNOW", "SNWD"]:
fv = round(distance(float(val), "MM").value("IN"), 1)
ret = None if fv < 0 else fv
return ret
def varconv(val, element):
"""Convert NCDC to something we use in the database"""
ret = None
if element in ['TMAX', 'TMIN']:
fv = round(temperature(float(val) / 10.0, 'C').value('F'), 0)
ret = None if (fv < -100 or fv > 150) else fv
elif element in ['PRCP']:
fv = round(distance(float(val) / 10.0, 'MM').value('IN'), 2)
ret = None if fv < 0 else fv
elif element in ['SNOW', 'SNWD']:
fv = round(distance(float(val), 'MM').value('IN'), 1)
ret = None if fv < 0 else fv
return ret
def estimate_snow(df, ts):
"""Estimate the Snow based on COOP reports"""
idx = iemre.daily_offset(ts)
nc = ncopen(iemre.get_daily_ncname(ts.year), 'r', timeout=300)
snowgrid12 = distance(nc.variables['snow_12z'][idx, :, :],
'MM').value('IN').filled(0)
snowdgrid12 = distance(nc.variables['snowd_12z'][idx, :, :],
'MM').value('IN').filled(0)
nc.close()
for sid, row in df.iterrows():
if pd.isnull(row['snow']):
df.at[sid, 'snow'] = snowgrid12[row['gridj'], row['gridi']]
if pd.isnull(row['snowd']):
df.at[sid, 'snowd'] = snowdgrid12[row['gridj'], row['gridi']]
def replace_forecast(df, location):
"""Replace dataframe data with forecast for this location"""
pgconn = psycopg2.connect(database='coop', host='iemdb', user='******')
cursor = pgconn.cursor()
today = datetime.date.today()
nextjan1 = datetime.date(today.year + 1, 1, 1)
coop = XREF[location]['climodat']
years = [int(y) for y in np.arange(df.index.values.min().year,
df.index.values.max().year + 1)]
cursor.execute("""
SELECT day, high, low, precip from alldata_forecast WHERE
modelid = (SELECT id from forecast_inventory WHERE model = 'NDFD'
ORDER by modelts DESC LIMIT 1) and station = %s and day >= %s
""", (coop, today))
rcols = ['maxt', 'mint', 'rain']
for row in cursor:
valid = row[0]
maxc = temperature(row[1], 'F').value('C')
minc = temperature(row[2], 'F').value('C')
rain = distance(row[3], 'IN').value('MM')
for year in years:
df.loc[valid.replace(year=year), rcols] = (maxc, minc, rain)
# Need to get radiation from CFS
cursor.execute("""
SELECT day, srad from alldata_forecast WHERE
modelid = (SELECT id from forecast_inventory WHERE model = 'CFS'
ORDER by modelts DESC LIMIT 1) and station = %s and day >= %s
and day < %s
""", (coop, today, nextjan1))
for row in cursor:
valid = row[0]
for year in years:
df.loc[valid.replace(year=year), 'radn'] = row[1]
def main(argv):
"""Do Stuff"""
fn = argv[1]
year = int(argv[2])
df = read_cli(fn)
df["pcpn_in"] = distance(df["pcpn"].values, "MM").value("IN")
do_qc(fn, df, year)
def load(dirname, location, sdate):
""" Read a file please """
data = []
idx = []
mindoy = int(sdate.strftime("%j"))
for line in open("%s/%s.met" % (dirname, location)):
line = line.strip()
if not line.startswith('19') and not line.startswith('20'):
continue
tokens = line.split()
if int(tokens[1]) < mindoy:
continue
data.append(tokens)
ts = (datetime.date(int(tokens[0]), 1, 1) +
datetime.timedelta(days=int(tokens[1])-1))
idx.append(ts)
if len(data[0]) < 10:
cols = ['year', 'doy', 'radn', 'maxt', 'mint', 'rain']
else:
cols = ['year', 'doy', 'radn', 'maxt', 'mint',
'rain', 'gdd', 'st4', 'st12', 'st24',
'st50', 'sm12', 'sm24', 'sm50']
df = pd.DataFrame(data, index=idx,
columns=cols)
for col in cols:
df[col] = pd.to_numeric(df[col], errors='coerce')
if len(data[0]) < 10:
df['gdd'] = gdd(temperature(df['maxt'].values, 'C'),
temperature(df['mint'].values, 'C'))
df['gddcum'] = df.groupby(['year'])['gdd'].apply(lambda x: x.cumsum())
df['raincum'] = distance(
df.groupby(['year'])['rain'].apply(lambda x: x.cumsum()),
'MM').value('IN')
return df
def replace_cfs(df, location):
"""Replace the CFS data for this year!"""
pgconn = get_dbconn('coop', user='******')
cursor = pgconn.cursor()
coop = XREF[location]['climodat']
today = datetime.date.today() + datetime.timedelta(days=3)
dec31 = today.replace(day=31, month=12)
cursor.execute(
"""
SELECT day, high, low, precip, srad from alldata_forecast WHERE
modelid = (SELECT id from forecast_inventory WHERE model = 'CFS'
ORDER by modelts DESC LIMIT 1) and station = %s and day >= %s
and day <= %s ORDER by day ASC
""", (coop, today, dec31))
rcols = ['maxt', 'mint', 'rain', 'radn']
if cursor.rowcount == 0:
print(" replace_cfs found zero rows!")
return
for row in cursor:
maxt = temperature(row[1], 'F').value('C')
mint = temperature(row[2], 'F').value('C')
rain = distance(row[3], 'IN').value('MM')
radn = row[4]
df.loc[row[0], rcols] = [maxt, mint, rain, radn]
if row[0] == dec31:
return
now = row[0] + datetime.timedelta(days=1)
# OK, if our last row does not equal dec31, we have some more work to do
print((" Replacing %s->%s with previous year's data") % (now, dec31))
while now <= dec31:
lastyear = now.replace(year=(now.year - 1))
df.loc[now, rcols] = df.loc[lastyear, rcols]
now += datetime.timedelta(days=1)
def do(lon, lat, station):
""" Process this station and geography """
idx = np.digitize([lon, ], lons)[0]
jdx = np.digitize([lat, ], lats)[0]
print("--> Processing %s i:%s j:%s" % (station, idx, jdx))
pdata = pr_nc.variables['pr'][:, jdx, idx]
xdata = tasmax_nc.variables['tmax'][:, jdx, idx]
ndata = tasmin_nc.variables['tmin'][:, jdx, idx]
highs = temperature(xdata, 'C').value('F')
lows = temperature(ndata, 'C').value('F')
precips = distance(pdata, 'MM').value('IN')
now = basets
high = low = precip = None
for k, _ in enumerate(tmdata):
now += datetime.timedelta(days=1)
if now.month == 2 and now.day == 29:
# Insert missing data
insert(station, now, high, low, precip)
now += datetime.timedelta(days=1)
high = fix(highs[k])
low = fix(lows[k])
if low is not None and high is not None and low > high:
# Swap, sigh
print(('%s %s high: %.1f low: %.1f was swapped'
) % (now.strftime("%m-%d-%Y"), station, high, low))
high2 = high
high = low
low = high2
precip = fix(precips[k])
insert(station, now, high, low, precip)
def main(argv):
"""Do Stuff"""
fn = argv[1]
year = int(argv[2])
df = read_cli(fn)
df['pcpn_in'] = distance(df['pcpn'].values, 'MM').value('IN')
do_qc(fn, df, year)
def dowork(form):
"""Do work!"""
date = datetime.datetime.strptime(form.getfirst('valid'), '%Y-%m-%d')
lat = float(form.getfirst("lat"))
lon = float(form.getfirst("lon"))
# We want data for the UTC date and timestamps are in the rears, so from
# 1z through 1z
sts = utc(date.year, date.month, date.day, 1)
ets = sts + datetime.timedelta(hours=24)
sidx = iemre.hourly_offset(sts)
eidx = iemre.hourly_offset(ets)
ncfn = "/mesonet/data/stage4/%s_stage4_hourly.nc" % (date.year, )
res = {'gridi': -1, 'gridj': -1, 'data': []}
if not os.path.isfile(ncfn):
return json.dumps(res)
with ncopen(ncfn) as nc:
dist = ((nc.variables['lon'][:] - lon)**2 +
(nc.variables['lat'][:] - lat)**2)**0.5
(j, i) = np.unravel_index(dist.argmin(), dist.shape)
res['gridi'] = int(i)
res['gridj'] = int(j)
ppt = nc.variables['p01m'][sidx:eidx, j, i]
for tx, pt in enumerate(ppt):
valid = sts + datetime.timedelta(hours=tx)
res['data'].append({
'end_valid': valid.strftime("%Y-%m-%dT%H:00:00Z"),
'precip_in': myrounder(
datatypes.distance(pt, 'MM').value('IN'), 2)
})
return json.dumps(res)
def compute_stage4(lon, lat, year):
"""Build a daily dataframe for the stage4 data"""
nc = netCDF4.Dataset("/mesonet/data/stage4/%s_stage4_hourly.nc" % (year,))
lons = nc.variables['lon'][:]
lats = nc.variables['lat'][:]
dist = ((lons - lon)**2 + (lats - lat)**2)**0.5
(yidx, xidx) = np.unravel_index(dist.argmin(), dist.shape)
print(("Computed stage4 nclon:%.2f nclat:%.2f yidx:%s xidx:%s "
) % (lons[yidx, xidx], lats[yidx, xidx], yidx, xidx))
p01i = distance(nc.variables['p01m'][:, yidx, xidx], 'MM').value('IN')
nc.close()
df = pd.DataFrame({'precip': 0.0},
index=pd.date_range('%s-01-01' % (year, ),
'%s-12-31' % (year, ),
tz='America/Chicago'))
for date in df.index.values:
date2 = datetime.datetime.utcfromtimestamp(date.tolist()/1e9)
ts = datetime.datetime(date2.year, date2.month, date2.day, 6)
ts = ts.replace(tzinfo=pytz.utc)
ts = ts.astimezone(pytz.timezone("America/Chicago"))
ts = ts.replace(hour=0)
ts = ts.astimezone(pytz.utc)
tidx = hourly_offset(ts)
# values are in the rears
val = np.ma.sum(p01i[tidx+1:tidx+25])
if val > 0:
df.at[date, 'precip'] = val # close enough
return df
def main(argv):
"""Do Stuff"""
fn = argv[1]
year = int(argv[2])
df = read_cli(fn)
df['pcpn_in'] = distance(df['pcpn'].values, 'MM').value('IN')
do_qc(fn, df, year)
def main():
"""Go!"""
title = 'NOAA MRMS Q3: RADAR + Guage Corrected Rainfall Estimates + NWS Storm Reports'
mp = MapPlot(sector='custom',
north=42.3, east=-93.0, south=41.65, west=-94.1,
axisbg='white',
titlefontsize=14,
title=title,
subtitle='Valid: 14 June 2018')
shp = shapefile.Reader('cities.shp')
for record in shp.shapeRecords():
geo = shape(record.shape)
mp.ax.add_geometries([geo], ccrs.PlateCarree(), zorder=Z_OVERLAY2,
facecolor='None', edgecolor='k', lw=2)
grbs = pygrib.open('MRMS_GaugeCorr_QPE_24H_00.00_20180614-200000.grib2')
grb = grbs.message(1)
pcpn = distance(grb['values'], 'MM').value('IN')
lats, lons = grb.latlons()
lons -= 360.
clevs = [0.01, 0.1, 0.3, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10]
cmap = nwsprecip()
cmap.set_over('k')
mp.pcolormesh(lons, lats, pcpn, clevs, cmap=cmap, latlon=True,
units='inch')
lons, lats, vals, labels = get_data()
mp.drawcounties()
mp.plot_values(lons, lats, vals, "%s", labels=labels,
labelbuffer=1, labelcolor='white')
mp.drawcities(labelbuffer=5, minarea=0.2)
mp.postprocess(filename='test.png')
def main():
"""Go Main Go"""
os.chdir("baseline")
for fn in glob.glob("*.met"):
location = fn[:-4]
output = open("%s.csv" % (location, ), "w")
output.write("date,high[F],low[F],precip[inch],gdd[F]\n")
for line in open(fn):
line = line.strip()
if (not line.startswith("2012") and not line.startswith("2015")
and not line.startswith("2016")):
continue
tokens = line.split()
valid = datetime.date(int(
tokens[0]), 1, 1) + datetime.timedelta(days=int(tokens[1]) - 1)
high = temperature(float(tokens[3]), "C")
low = temperature(float(tokens[4]), "C")
gdd = met.gdd(high, low, 50, 86)
precip = distance(float(tokens[5]), "MM")
output.write(("%s,%.1f,%.1f,%.2f,%.1f\n") % (
valid.strftime("%Y-%m-%d"),
high.value("F"),
low.value("F"),
precip.value("IN"),
gdd,
))
output.close()
def one():
"""option 1"""
icursor = ISUAG.cursor()
iemcursor = IEM.cursor()
icursor.execute(
"""
SELECT station, valid, ws_mps_s_wvt, winddir_d1_wvt, rain_mm_tot,
tair_c_max, tair_c_min
from sm_daily
"""
)
for row in icursor:
avg_sknt = speed(row[2], "MPS").value("KT")
avg_drct = row[3]
pday = distance(row[4], "MM").value("IN")
high = temperature(row[5], "C").value("F")
low = temperature(row[6], "C").value("F")
iemcursor.execute(
"""
UPDATE summary SET avg_sknt = %s, vector_avg_drct = %s, pday = %s,
max_tmpf = %s, min_tmpf = %s
WHERE
iemid = (select iemid from stations WHERE network = 'ISUSM' and
id = %s) and day = %s
""",
(avg_sknt, avg_drct, pday, high, low, row[0], row[1]),
)
iemcursor.close()
IEM.commit()
IEM.close()
def compute_stage4(lon, lat, year):
"""Build a daily dataframe for the stage4 data"""
nc = netCDF4.Dataset("/mesonet/data/stage4/%s_stage4_hourly.nc" % (year, ))
lons = nc.variables['lon'][:]
lats = nc.variables['lat'][:]
dist = ((lons - lon)**2 + (lats - lat)**2)**0.5
(yidx, xidx) = np.unravel_index(dist.argmin(), dist.shape)
print(("Computed stage4 nclon:%.2f nclat:%.2f yidx:%s xidx:%s ") %
(lons[yidx, xidx], lats[yidx, xidx], yidx, xidx))
p01i = distance(nc.variables['p01m'][:, yidx, xidx], 'MM').value('IN')
nc.close()
df = pd.DataFrame({'precip': 0.0},
index=pd.date_range('%s-01-01' % (year, ),
'%s-12-31' % (year, ),
tz='America/Chicago'))
for date in df.index.values:
date2 = datetime.datetime.utcfromtimestamp(date.tolist() / 1e9)
ts = datetime.datetime(date2.year, date2.month, date2.day, 6)
ts = ts.replace(tzinfo=pytz.utc)
ts = ts.astimezone(pytz.timezone("America/Chicago"))
ts = ts.replace(hour=0)
ts = ts.astimezone(pytz.utc)
tidx = hourly_offset(ts)
# values are in the rears
val = np.ma.sum(p01i[tidx + 1:tidx + 25])
if val > 0:
df.at[date, 'precip'] = val # close enough
return df
def grid_day(nc, ts):
"""
I proctor the gridding of data on an hourly basis
@param ts Timestamp of the analysis, we'll consider a 20 minute window
"""
cursor = COOP.cursor(cursor_factory=psycopg2.extras.DictCursor)
offset = iemre.daily_offset(ts)
if ts.day == 29 and ts.month == 2:
ts = datetime.datetime(2000, 3, 1)
sql = """SELECT * from climate51 WHERE valid = '%s' and
substr(station,3,4) != '0000' and substr(station,3,1) != 'C'
""" % (ts.strftime("%Y-%m-%d"), )
cursor.execute(sql)
res = generic_gridder(nc, cursor, "high")
nc.variables["high_tmpk"][offset] = datatypes.temperature(res,
"F").value("K")
cursor.scroll(0, mode="absolute")
res = generic_gridder(nc, cursor, "low")
nc.variables["low_tmpk"][offset] = datatypes.temperature(res,
"F").value("K")
cursor.scroll(0, mode="absolute")
res = generic_gridder(nc, cursor, "precip")
nc.variables["p01d"][offset] = datatypes.distance(res, "IN").value("MM")
cursor.scroll(0, mode="absolute")
res = generic_gridder(nc, cursor, "gdd50")
nc.variables["gdd50"][offset] = res
def dowork(form):
"""Do work!"""
date = datetime.datetime.strptime(form.getfirst('valid'), '%Y-%m-%d')
lat = float(form.getfirst("lat"))
lon = float(form.getfirst("lon"))
# We want data for the UTC date and timestamps are in the rears, so from
# 1z through 1z
sts = datetime.datetime(date.year, date.month, date.day, 1)
sts = sts.replace(tzinfo=pytz.utc)
ets = sts + datetime.timedelta(hours=24)
sidx = iemre.hourly_offset(sts)
eidx = iemre.hourly_offset(ets)
ncfn = "/mesonet/data/stage4/%s_stage4_hourly.nc" % (date.year, )
nc = netCDF4.Dataset(ncfn, 'r')
dist = ((nc.variables['lon'][:] - lon)**2 +
(nc.variables['lat'][:] - lat)**2)**0.5
(j, i) = np.unravel_index(dist.argmin(), dist.shape)
res = {'gridi': i, 'gridj': j, 'data': []}
ppt = nc.variables['p01m'][sidx:eidx, j, i]
nc.close()
for tx, pt in enumerate(ppt):
valid = sts + datetime.timedelta(hours=tx)
res['data'].append({
'end_valid':
valid.strftime("%Y-%m-%dT%H:00:00Z"),
'precip_in':
myrounder(datatypes.distance(pt, 'MM').value('IN'), 2)
})
return json.dumps(res)
def main():
"""Go Main Go"""
fh = open("insert.sql", "w")
req = requests.get("http://www.weather.gov/source/aprfc/nrcs_swe.json")
j = req.json()
for feature in j["features"]:
nwsli = feature["properties"]["lid"]
name = feature["properties"]["name"]
elev = distance(float(feature["properties"]["elev"]), "FT").value("M")
lon, lat = feature["geometry"]["coordinates"]
country = nwsli2country.get(nwsli[3:])
state = nwsli2state.get(nwsli[3:])
network = "%s_DCP" % (state, )
sql = """INSERT into stations(id, name, network, country, state,
plot_name, elevation, online, metasite, geom) VALUES ('%s', '%s', '%s',
'%s', '%s', '%s', %s, 't', 'f', 'SRID=4326;POINT(%s %s)');
""" % (
nwsli,
name,
network,
country,
state,
name,
float(elev),
float(lon),
float(lat),
)
fh.write(sql)
fh.close()
def do_month(year, month, routes):
""" Generate a MRMS plot for the month!"""
sts = datetime.datetime(year, month, 1)
ets = sts + datetime.timedelta(days=35)
ets = ets.replace(day=1)
today = datetime.datetime.now()
if ets > today:
ets = today
idx0 = iemre.daily_offset(sts)
idx1 = iemre.daily_offset(ets)
nc = ncopen(iemre.get_daily_mrms_ncname(year), 'r')
lats = nc.variables['lat'][:]
lons = nc.variables['lon'][:]
p01d = distance(np.sum(nc.variables['p01d'][idx0:idx1, :, :], 0),
'MM').value('IN')
nc.close()
mp = MapPlot(sector='iowa', title='MRMS %s - %s Total Precipitation' % (
sts.strftime("%-d %b"),
(ets - datetime.timedelta(days=1)).strftime("%-d %b %Y")),
subtitle='Data from NOAA MRMS Project')
x, y = np.meshgrid(lons, lats)
bins = [0.01, 0.1, 0.5, 1, 1.5, 2, 3, 4, 5, 6, 7, 8, 12, 16, 20]
mp.pcolormesh(x, y, p01d, bins, units='inches')
mp.drawcounties()
currentfn = "summary/iowa_mrms_q3_month.png"
archivefn = sts.strftime("%Y/%m/summary/iowa_mrms_q3_month.png")
pqstr = "plot %s %s00 %s %s png" % (
routes, sts.strftime("%Y%m%d%H"), currentfn, archivefn)
mp.postprocess(pqstr=pqstr)
def load(dirname, location, sdate):
""" Read a file please """
data = []
idx = []
mindoy = int(sdate.strftime("%j"))
for line in open("%s/%s.met" % (dirname, location)):
line = line.strip()
if not line.startswith('19') and not line.startswith('20'):
continue
tokens = line.split()
if int(tokens[1]) < mindoy:
continue
data.append(tokens)
ts = (datetime.date(int(tokens[0]), 1, 1) +
datetime.timedelta(days=int(tokens[1])-1))
idx.append(ts)
if len(data[0]) < 10:
cols = ['year', 'doy', 'radn', 'maxt', 'mint', 'rain']
else:
cols = ['year', 'doy', 'radn', 'maxt', 'mint',
'rain', 'gdd', 'st4', 'st12', 'st24',
'st50', 'sm12', 'sm24', 'sm50']
df = pd.DataFrame(data, index=idx,
columns=cols)
for col in cols:
df[col] = pd.to_numeric(df[col], errors='coerce')
if len(data[0]) < 10:
df['gdd'] = gdd(temperature(df['maxt'].values, 'C'),
temperature(df['mint'].values, 'C'))
df['gddcum'] = df.groupby(['year'])['gdd'].apply(lambda x: x.cumsum())
df['raincum'] = distance(
df.groupby(['year'])['rain'].apply(lambda x: x.cumsum()),
'MM').value('IN')
return df
def grid_day(nc, ts):
"""
I proctor the gridding of data on an hourly basis
@param ts Timestamp of the analysis, we'll consider a 20 minute window
"""
cursor = COOP.cursor(cursor_factory=psycopg2.extras.DictCursor)
offset = iemre.daily_offset(ts)
if ts.day == 29 and ts.month == 2:
ts = datetime.datetime(2000, 3, 1)
sql = """SELECT * from ncdc_climate71 WHERE valid = '%s' and
substr(station,3,4) != '0000' and substr(station,3,1) != 'C'
""" % (ts.strftime("%Y-%m-%d"), )
cursor.execute(sql)
if cursor.rowcount > 4:
res = generic_gridder(nc, cursor, 'high')
if res is not None:
nc.variables['tmax'][offset] = datatypes.temperature(
res, 'F').value('C')
cursor.scroll(0, mode='absolute')
res = generic_gridder(nc, cursor, 'low')
if res is not None:
nc.variables['tmin'][offset] = datatypes.temperature(
res, 'F').value('C')
cursor.scroll(0, mode='absolute')
res = generic_gridder(nc, cursor, 'precip')
if res is not None:
nc.variables['ppt'][offset] = datatypes.distance(res,
'IN').value('MM')
else:
print(("%s has %02i entries, FAIL") %
(ts.strftime("%Y-%m-%d"), cursor.rowcount))
def main():
"""Go Main"""
nt = NetworkTable(['AWOS', 'IA_ASOS'])
pgconn = psycopg2.connect(database='mos', host='localhost', user='******',
port=5555)
df = read_sql("""
select station, avg(pwater) from model_gridpoint where
model = 'NAM' and extract(hour from runtime at time zone 'UTC') = 0
and pwater > 0 and pwater < 100 and
extract(month from runtime) between 4 and 9 and ftime = runtime
GROUP by station ORDER by avg
""", pgconn, index_col='station')
df['lat'] = None
df['lon'] = None
df['pwater'] = distance(df['avg'].values, 'MM').value('IN')
for station in df.index.values:
df.at[station, 'lat'] = nt.sts[station[1:]]['lat']
df.at[station, 'lon'] = nt.sts[station[1:]]['lon']
mp = MapPlot(title=('00z Analysis NAM Warm-Season Average '
'Precipitable Water [in]'),
subtitle=("based on grid point samples "
"from 2004-2017 (April-September)"))
cmap = plt.get_cmap("plasma_r")
cmap.set_under('white')
cmap.set_over('black')
mp.contourf(df['lon'], df['lat'], df['pwater'],
np.arange(0.94, 1.13, 0.03), cmap=cmap,
units='inch')
mp.drawcounties()
mp.drawcities()
mp.postprocess(filename='170901.png')
mp.close()
def replace_forecast(df, location):
"""Replace dataframe data with forecast for this location"""
pgconn = psycopg2.connect(database='coop', host='iemdb', user='******')
cursor = pgconn.cursor()
today = datetime.date.today()
nextjan1 = datetime.date(today.year + 1, 1, 1)
coop = XREF[location]['climodat']
years = [int(y) for y in np.arange(df.index.values.min().year,
df.index.values.max().year + 1)]
cursor.execute("""
SELECT day, high, low, precip from alldata_forecast WHERE
modelid = (SELECT id from forecast_inventory WHERE model = 'NDFD'
ORDER by modelts DESC LIMIT 1) and station = %s and day >= %s
""", (coop, today))
rcols = ['maxt', 'mint', 'rain']
for row in cursor:
valid = row[0]
maxc = temperature(row[1], 'F').value('C')
minc = temperature(row[2], 'F').value('C')
rain = distance(row[3], 'IN').value('MM')
for year in years:
df.loc[valid.replace(year=year), rcols] = (maxc, minc, rain)
# Need to get radiation from CFS
cursor.execute("""
SELECT day, srad from alldata_forecast WHERE
modelid = (SELECT id from forecast_inventory WHERE model = 'CFS'
ORDER by modelts DESC LIMIT 1) and station = %s and day >= %s
and day < %s
""", (coop, today, nextjan1))
for row in cursor:
valid = row[0]
for year in years:
df.loc[valid.replace(year=year), 'radn'] = row[1]
def daily_process(nwsli, maxts):
""" Process the daily file """
fn = "%s/%s_DailySI.dat" % (BASE, STATIONS[nwsli])
df = common_df_logic(fn, maxts, nwsli, "sm_daily")
if df is None:
return 0
LOG.debug("processing %s rows from %s", len(df.index), fn)
processed = 0
acursor = ACCESS.cursor()
for _i, row in df.iterrows():
# Need a timezone
valid = datetime.datetime(
row["valid"].year, row["valid"].month, row["valid"].day, 12, 0
)
valid = valid.replace(tzinfo=pytz.timezone("America/Chicago"))
ob = Observation(nwsli, "ISUSM", valid)
ob.data["max_tmpf"] = temperature(row["tair_c_max_qc"], "C").value("F")
ob.data["min_tmpf"] = temperature(row["tair_c_min_qc"], "C").value("F")
ob.data["pday"] = round(
distance(row["rain_mm_tot_qc"], "MM").value("IN"), 2
)
if valid not in EVENTS["days"]:
EVENTS["days"].append(valid)
ob.data["et_inch"] = distance(row["dailyet_qc"], "MM").value("IN")
ob.data["srad_mj"] = row["slrmj_tot_qc"]
# Someday check if this is apples to apples here
ob.data["vector_avg_drct"] = row["winddir_d1_wvt_qc"]
if ob.data["max_tmpf"] is None:
EVENTS["reprocess_temps"] = True
if ob.data["srad_mj"] == 0 or np.isnan(ob.data["srad_mj"]):
LOG.info(
"soilm_ingest.py station: %s ts: %s has 0 solar",
nwsli,
valid.strftime("%Y-%m-%d"),
)
EVENTS["reprocess_solar"] = True
if "ws_mps_max" in df.columns:
ob.data["max_sknt"] = speed(row["ws_mps_max_qc"], "MPS").value(
"KT"
)
ob.data["avg_sknt"] = speed(row["ws_mps_s_wvt_qc"], "MPS").value("KT")
ob.save(acursor)
processed += 1
acursor.close()
ACCESS.commit()
return processed
def do_var(varname):
"""
Run our estimator for a given variable
"""
currentnc = None
sql = """select day, station from alldata_%s WHERE %s is null
and day >= '1893-01-01' ORDER by day ASC""" % (state.lower(), varname)
ccursor.execute(sql)
for row in ccursor:
day = row[0]
station = row[1]
if station not in nt.sts:
continue
sql = """
SELECT station, %s from alldata_%s WHERE %s is not NULL
and station in %s and day = '%s'
""" % (varname, state, varname, tuple(friends[station]), day)
ccursor2.execute(sql)
weight = []
value = []
for row2 in ccursor2:
idx = friends[station].index(row2[0])
weight.append(weights[station][idx])
value.append(row2[1])
if len(weight) < 3:
# Nearest neighbors failed, so lets look at our grided analysis
# and sample from it
if currentnc is None or currentnc.title.find(str(day.year)) == -1:
currentnc = netCDF4.Dataset(("/mesonet/data/iemre/"
"%s_mw_daily.nc") % (day.year,))
tidx = iemre.daily_offset(datetime.datetime(day.year, day.month,
day.day))
iidx, jidx = iemre.find_ij(nt.sts[station]['lon'],
nt.sts[station]['lat'])
iemreval = currentnc.variables[vnameconv[varname]][tidx, jidx,
iidx]
if varname in ('high', 'low'):
interp = temperature(iemreval, 'K').value('F')
else:
interp = distance(iemreval, 'MM').value('IN')
print '--> Neighbor failure, %s %s %s' % (station, day, varname)
else:
mass = sum(weight)
interp = np.sum(np.array(weight) * np.array(value) / mass)
dataformat = '%.2f'
if varname in ['high', 'low']:
dataformat = '%.0f'
print(('Set station: %s day: %s varname: %s value: %s'
) % (station, day, varname, dataformat % (interp,)))
sql = """
UPDATE alldata_%s SET estimated = true, %s = %s WHERE
station = '%s' and day = '%s'
""" % (state.lower(), varname,
dataformat % (interp,), station, day)
sql = sql.replace(' nan ', ' null ')
ccursor2.execute(sql)
def doday(ts, realtime):
"""
Create a plot of precipitation stage4 estimates for some day
"""
lts = utc(ts.year, ts.month, ts.day, 12)
lts = lts.astimezone(pytz.timezone("America/Chicago"))
# make assumptions about the last valid MRMS data
if realtime:
# Up until :59 after of the last hour
lts = (datetime.datetime.now() -
datetime.timedelta(hours=1)).replace(minute=59)
else:
lts = lts.replace(year=ts.year,
month=ts.month,
day=ts.day,
hour=23,
minute=59)
idx = iemre.daily_offset(ts)
ncfn = iemre.get_daily_mrms_ncname(ts.year)
if not os.path.isfile(ncfn):
LOG.info("File %s missing, abort.", ncfn)
return
with ncopen(ncfn, timeout=300) as nc:
precip = nc.variables['p01d'][idx, :, :]
lats = nc.variables['lat'][:]
lons = nc.variables['lon'][:]
subtitle = "Total between 12:00 AM and %s" % (
lts.strftime("%I:%M %p %Z"), )
routes = 'ac'
if not realtime:
routes = 'a'
# clevs = np.arange(0, 0.25, 0.05)
# clevs = np.append(clevs, np.arange(0.25, 3., 0.25))
# clevs = np.append(clevs, np.arange(3., 10.0, 1))
clevs = [
0.01, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10
]
(xx, yy) = np.meshgrid(lons, lats)
for sector in ['iowa', 'midwest']:
pqstr = ("plot %s %s00 %s_q2_1d.png %s_q2_1d.png png") % (
routes, ts.strftime("%Y%m%d%H"), sector, sector)
mp = MapPlot(title=("%s NCEP MRMS Q3 Today's Precipitation") %
(ts.strftime("%-d %b %Y"), ),
subtitle=subtitle,
sector=sector)
mp.pcolormesh(xx,
yy,
distance(precip, 'MM').value('IN'),
clevs,
cmap=nwsprecip(),
units='inch')
if sector == 'iowa':
mp.drawcounties()
mp.postprocess(pqstr=pqstr, view=False)
mp.close()
def plotter(fdict):
""" Go """
ctx = get_autoplot_context(fdict, get_description())
csector = ctx['csector']
date = ctx['date']
z = ctx['z']
period = ctx['f']
scale = ctx['scale']
valid = utc(date.year, date.month, date.day, int(z))
gribfn = valid.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/model/wpc/"
"p" + period + "m_%Y%m%d%Hf" + period + ".grb"))
if not os.path.isfile(gribfn):
raise ValueError("gribfn %s missing" % (gribfn, ))
grbs = pygrib.open(gribfn)
grb = grbs[1]
precip = distance(grb.values, 'MM').value('IN')
lats, lons = grb.latlons()
title = ("Weather Prediction Center %s Quantitative "
"Precipitation Forecast") % (PDICT[period])
subtitle = ("%sWPC Forcast %s UTC to %s UTC") % (
("US Drought Monitor Overlaid, " if ctx['opt'] == 'both' else ''),
valid.strftime("%d %b %Y %H"),
(valid +
datetime.timedelta(hours=int(period))).strftime("%d %b %Y %H"))
mp = MapPlot(sector=('state' if len(csector) == 2 else csector),
state=ctx['csector'],
title=title,
subtitle=subtitle,
continentalcolor='white',
titlefontsize=16)
cmap = plt.get_cmap('gist_ncar')
cmap.set_under('#EEEEEE')
cmap.set_over('black')
if scale == 'auto':
levs = np.linspace(0, np.max(precip) * 1.1, 10)
levs = [round(lev, 2) for lev in levs]
levs[0] = 0.01
elif scale == '10':
levs = np.arange(0, 10.1, 1.)
levs[0] = 0.01
elif scale == '7':
levs = np.arange(0, 7.1, 0.5)
levs[0] = 0.01
elif scale == '3.5':
levs = np.arange(0, 3.6, 0.25)
levs[0] = 0.01
mp.pcolormesh(lons,
lats,
precip,
levs,
cmap=cmap,
units='inch',
clip_on=False)
if ctx['opt'] == 'both':
mp.draw_usdm(valid=valid, filled=False, hatched=True)
return mp.fig
def replace_obs_iem(df, location):
"""Replace dataframe data with obs for this location
Tricky part, if the baseline already provides data for this year, we should
use it!
"""
pgconn = psycopg2.connect(database='iem', host='iemdb', user='******')
cursor = pgconn.cursor()
station = XREF[location]['station']
today = datetime.date.today()
jan1 = today.replace(month=1, day=1)
years = [int(y) for y in np.arange(df.index.values.min().year,
df.index.values.max().year + 1)]
table = "summary_%s" % (jan1.year,)
cursor.execute("""
select day, max_tmpf, min_tmpf, srad_mj, pday
from """ + table + """ s JOIN stations t on (s.iemid = t.iemid)
WHERE t.id = %s and max_tmpf is not null
and day < 'TODAY' ORDER by day ASC
""", (station,))
rcols = ['maxt', 'mint', 'radn', 'gdd', 'rain']
replaced = []
for row in cursor:
valid = row[0]
# Does our df currently have data for this date? If so, we shall do
# no more
dont_replace = not np.isnan(df.at[valid, 'mint'])
if not dont_replace:
replaced.append(valid)
_gdd = gdd(temperature(row[1], 'F'), temperature(row[2], 'F'))
for year in years:
if valid.month == 2 and valid.day == 29 and year % 4 != 0:
continue
if dont_replace:
df.loc[valid.replace(year=year),
rcols[3:-1]] = (_gdd,
distance(row[4], 'in').value('mm'))
continue
df.loc[valid.replace(year=year), rcols] = (
temperature(row[1], 'F').value('C'),
temperature(row[2], 'F').value('C'), row[3],
_gdd, distance(row[4], 'in').value('mm'))
if len(replaced) > 0:
print((" used IEM Access %s from %s->%s"
) % (station, replaced[0], replaced[-1]))
def doday(ts, realtime):
"""
Create a plot of precipitation stage4 estimates for some day
We should total files from 1 AM to midnight local time
"""
sts = ts.replace(hour=1)
ets = sts + datetime.timedelta(hours=24)
interval = datetime.timedelta(hours=1)
now = sts
total = None
lts = None
while now < ets:
gmt = now.astimezone(pytz.timezone("UTC"))
fn = gmt.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/"
"stage4/ST4.%Y%m%d%H.01h.grib"))
if os.path.isfile(fn):
lts = now
grbs = pygrib.open(fn)
if total is None:
g = grbs[1]
total = g["values"]
lats, lons = g.latlons()
else:
total += grbs[1]["values"]
grbs.close()
now += interval
if lts is None and ts.hour > 1:
print 'stage4_today_total.py found no data!'
if lts is None:
return
lts = lts - datetime.timedelta(minutes=1)
subtitle = "Total between 12:00 AM and %s" % (lts.strftime("%I:%M %p %Z"),)
routes = 'ac'
if not realtime:
routes = 'a'
for sector in ['iowa', 'midwest', 'conus']:
pqstr = ("plot %s %s00 %s_stage4_1d.png %s_stage4_1d.png png"
) % (routes,
ts.strftime("%Y%m%d%H"), sector, sector)
m = MapPlot(sector=sector,
title="%s NCEP Stage IV Today's Precipitation" % (
ts.strftime("%-d %b %Y"),),
subtitle=subtitle)
clevs = [0.01, 0.1, 0.3, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10]
m.pcolormesh(lons, lats, distance(total, 'MM').value('IN'), clevs,
cmap=nwsprecip(), units='inch')
# map.drawstates(zorder=2)
if sector == 'iowa':
m.drawcounties()
m.postprocess(pqstr=pqstr)
m.close()
def do(ts, hours):
"""
Create a plot of precipitation stage4 estimates for some day
"""
ts = ts.replace(minute=0)
sts = ts - datetime.timedelta(hours=hours)
ets = ts
interval = datetime.timedelta(hours=1)
now = sts
total = None
lts = None
while now < ets:
fn = ("/mesonet/ARCHIVE/data/%s/stage4/ST4.%s.01h.grib") % (
now.strftime("%Y/%m/%d"), now.strftime("%Y%m%d%H"))
if os.path.isfile(fn):
lts = now
grbs = pygrib.open(fn)
if total is None:
g = grbs[1]
total = g["values"].filled(0)
lats, lons = g.latlons()
else:
total += grbs[1]["values"].filled(0)
grbs.close()
now += interval
if lts is None and ts.hour > 1:
print 'Missing StageIV data!'
if lts is None:
return
cmap = cm.get_cmap("jet")
cmap.set_under('white')
cmap.set_over('black')
clevs = [0.01, 0.1, 0.25, 0.5, 1, 2, 3, 5, 8, 9.9]
localtime = (ts - datetime.timedelta(minutes=1)).astimezone(
pytz.timezone("America/Chicago"))
for sector in ['iowa', 'midwest', 'conus']:
m = MapPlot(sector=sector,
title='NCEP Stage IV %s Hour Precipitation' % (hours, ),
subtitle='Total up to %s' %
(localtime.strftime("%d %B %Y %I %p %Z"), ))
m.pcolormesh(lons,
lats,
distance(total, 'MM').value('IN'),
clevs,
units='inch')
pqstr = "plot %s %s00 %s_stage4_%sh.png %s_stage4_%sh_%s.png png" % (
'ac', ts.strftime("%Y%m%d%H"), sector, hours, sector, hours,
ts.strftime("%H"))
if sector == 'iowa':
m.drawcounties()
m.postprocess(pqstr=pqstr)
m.close()
def daily_process(nwsli, maxts):
""" Process the daily file """
# print '-------------- DAILY PROCESS ----------------'
fn = "%s/%s_DailySI.dat" % (BASE, STATIONS[nwsli])
df = common_df_logic(fn, maxts, nwsli, "sm_daily")
if df is None:
return 0
LOG.debug("processing %s rows from %s", len(df.index), fn)
processed = 0
acursor = ACCESS.cursor()
for _i, row in df.iterrows():
# Need a timezone
valid = datetime.datetime(row['valid'].year, row['valid'].month,
row['valid'].day, 12, 0)
valid = valid.replace(tzinfo=pytz.timezone("America/Chicago"))
ob = Observation(nwsli, 'ISUSM', valid)
ob.data['max_tmpf'] = temperature(row['tair_c_max_qc'], 'C').value('F')
ob.data['min_tmpf'] = temperature(row['tair_c_min_qc'], 'C').value('F')
ob.data['pday'] = round(distance(row['rain_mm_tot_qc'],
'MM').value('IN'), 2)
if valid not in EVENTS['days']:
EVENTS['days'].append(valid)
ob.data['et_inch'] = distance(row['dailyet_qc'], 'MM').value('IN')
ob.data['srad_mj'] = row['slrmj_tot_qc']
# Someday check if this is apples to apples here
ob.data['vector_avg_drct'] = row['winddir_d1_wvt_qc']
if ob.data['max_tmpf'] is None:
EVENTS['reprocess_temps'] = True
if ob.data['srad_mj'] == 0 or np.isnan(ob.data['srad_mj']):
LOG.info(
"soilm_ingest.py station: %s ts: %s has 0 solar",
nwsli, valid.strftime("%Y-%m-%d")
)
EVENTS['reprocess_solar'] = True
if 'ws_mps_max' in df.columns:
ob.data['max_sknt'] = speed(row['ws_mps_max_qc'],
'MPS').value('KT')
ob.data['avg_sknt'] = speed(row['ws_mps_s_wvt_qc'], 'MPS').value('KT')
ob.save(acursor)
processed += 1
acursor.close()
ACCESS.commit()
return processed
def load(dirname, location, sdate):
""" Read a file please """
data = []
idx = []
for line in open("%s/%s.met" % (dirname, location)):
line = line.strip()
if not line.startswith("19") and not line.startswith("20"):
continue
tokens = line.split()
data.append(tokens)
ts = datetime.date(int(tokens[0]), 1, 1) + datetime.timedelta(days=int(tokens[1]) - 1)
idx.append(ts)
if len(data[0]) < 10:
cols = ["year", "doy", "radn", "maxt", "mint", "rain"]
else:
cols = [
"year",
"doy",
"radn",
"maxt",
"mint",
"rain",
"gdd",
"st4",
"st12",
"st24",
"st50",
"sm12",
"sm24",
"sm50",
]
df = pd.DataFrame(data, index=idx, columns=cols)
for col in cols:
df[col] = pd.to_numeric(df[col], errors="coerce")
if len(data[0]) < 10:
df["gdd"] = gdd(temperature(df["maxt"].values, "C"), temperature(df["mint"].values, "C"))
bins = []
today = datetime.date.today()
for valid, _ in df.iterrows():
if valid >= today:
bins.append(0)
continue
if sdate == "nov1" and valid.month >= 11:
bins.append(valid.year + 1)
continue
if valid.month < today.month:
bins.append(valid.year)
continue
if valid.month == today.month and valid.day < today.day:
bins.append(valid.year)
continue
bins.append(0)
df["bin"] = bins
df["rain"] = distance(df["rain"].values, "MM").value("IN")
df["avgt"] = temperature((df["maxt"] + df["mint"]) / 2.0, "C").value("F")
return df
def estimate_snow(ts):
"""Estimate the Snow based on COOP reports"""
idx = iemre.daily_offset(ts)
nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_daily.nc" % (ts.year, ),
'r')
nc.set_auto_mask(True)
snowgrid12 = distance(nc.variables['snow_12z'][idx, :, :],
'MM').value('IN')
snowdgrid12 = distance(nc.variables['snowd_12z'][idx, :, :],
'MM').value('IN')
nc.close()
for sid in nt.sts:
val = snowgrid12[nt.sts[sid]['gridj'], nt.sts[sid]['gridi']]
if val >= 0 and val < 100:
nt.sts[sid]['snow'] = "%.1f" % (val, )
val = snowdgrid12[nt.sts[sid]['gridj'], nt.sts[sid]['gridi']]
if val >= 0 and val < 140:
nt.sts[sid]['snowd'] = "%.1f" % (val, )
def test_distance_conv():
""" Speed distance from M to MI to FT to SM to KM """
mi = datatypes.distance(1.0, "mi")
assert abs(1609.344 - mi.value("M")) < 0.1
assert abs(5280.0 - mi.value("FT")) < 0.1
assert abs(1.0 - mi.value("SM")) < 0.1
assert abs(1.61 - mi.value("KM")) < 0.01
assert abs(63360.00 - mi.value("IN")) < 0.01
assert abs(1609344.00 - mi.value("mm")) < 0.01
assert abs(160934.40 - mi.value("CM")) < 0.01
def test_distance_conv():
""" Speed distance from M to MI to FT to SM to KM """
mi = datatypes.distance(1.0, 'mi')
assert abs(1609.344 - mi.value('M')) < 0.1
assert abs(5280.0 - mi.value('FT')) < 0.1
assert abs(1.0 - mi.value('SM')) < 0.1
assert abs(1.61 - mi.value('KM')) < 0.01
assert abs(63360.00 - mi.value('IN')) < 0.01
assert abs(1609344.00 - mi.value('mm')) < 0.01
assert abs(160934.40 - mi.value('CM')) < 0.01
def test_distance_conv(self):
""" Speed distance from M to MI to FT to SM to KM """
mi = datatypes.distance(1.0, 'MI')
self.assertAlmostEquals(1609.344, mi.value('M'), 1)
self.assertAlmostEquals(5280.0, mi.value('FT'), 1)
self.assertAlmostEquals(1.0, mi.value('SM'), 1)
self.assertAlmostEquals(1.61, mi.value('KM'), 2)
self.assertAlmostEquals(63360.00, mi.value('IN'), 2)
self.assertAlmostEquals(1609344.00, mi.value('MM'), 2)
self.assertAlmostEquals(160934.40, mi.value('CM'), 2)
def one():
icursor.execute("""SELECT station, valid, rain_mm_tot, tair_c_max,
tair_c_min from sm_daily WHERE tair_c_max is not null""")
for row in icursor:
high = temperature(row[3], 'C').value('F')
low = temperature(row[4], 'C').value('F')
iemcursor.execute("""
SELECT pday, max_tmpf, min_tmpf from summary s JOIN stations t on
(t.iemid = s.iemid)
WHERE day = %s and t.id = %s and t.network = 'ISUSM'
""", (row[1], row[0]))
if iemcursor.rowcount == 0:
print 'Adding summary_%s row %s %s' % (row[1].year, row[0], row[1])
iemcursor.execute("""
INSERT into summary_""" + str(row[1].year) + """
(iemid, day, pday, max_tmpf, min_tmpf) VALUES (
(SELECT iemid from stations where id = '%s' and
network = 'ISUSM'), '%s', %s, %s, %s)
""" % (row[0], row[1], distance(row[2], 'MM').value('IN'), high,
low))
else:
row2 = iemcursor.fetchone()
if (row2[1] is None or row2[2] is None or
round(row2[0], 2) != round((distance(row[2],
'MM').value('IN')),
2) or
round(high, 2) != round(row2[1], 2) or
round(low, 2) != round(row2[2], 2)):
print(('Mismatch %s %s old: %s new: %s'
) % (row[0], row[1], row2[0],
distance(row[2], 'MM').value('IN')))
iemcursor.execute("""
UPDATE summary SET pday = %s, max_tmpf = %s,
min_tmpf = %s WHERE
iemid = (select iemid from stations WHERE network = 'ISUSM' and
id = %s) and day = %s
""", (distance(row[2], 'MM').value('IN'), high, low, row[0],
row[1]))
def compute_loc(self, loc, dist, bearing):
""" Figure out the lon/lat for this location """
lat = self.nwsli_provider[loc]['lat']
lon = self.nwsli_provider[loc]['lon']
# shortcut
if dist == 0:
return lon, lat
meters = distance(float(dist), "MI").value("M")
northing = meters * math.cos(math.radians(bearing)) / 111111.0
easting = (meters * math.sin(math.radians(bearing)) /
math.cos(math.radians(lat)) / 111111.0)
return lon + easting, lat + northing
def main():
"""Go Main Go"""
nc = ncopen("/mesonet/data/iemre/1979_narr.nc")
data = np.sum(nc.variables['apcp'][:, :, :], axis=0)
m = MapPlot(sector='conus', axisbg='tan',
title=(""),
subtitle='',
titlefontsize=16)
t = distance(data, 'MM').value('IN')
m.pcolormesh(nc.variables['lon'][:], nc.variables['lat'][:], t,
np.arange(0, 60, 2), units='F')
m.postprocess(filename='test.png')
def doday(ts, realtime):
"""
Create a plot of precipitation stage4 estimates for some day
"""
lts = datetime.datetime.utcnow().replace(
tzinfo=pytz.timezone("UTC"))
lts = lts.astimezone(pytz.timezone("America/Chicago"))
# make assumptions about the last valid MRMS data
if realtime:
# Up until :59 after of the last hour
lts = (lts - datetime.timedelta(hours=1)).replace(minute=59)
else:
lts = lts.replace(year=ts.year, month=ts.month, day=ts.day,
hour=23, minute=59)
idx = iemre.daily_offset(ts)
ncfn = "/mesonet/data/iemre/%s_mw_mrms_daily.nc" % (ts.year,)
nc = netCDF4.Dataset(ncfn)
precip = nc.variables['p01d'][idx, :, :]
lats = nc.variables['lat'][:]
lons = nc.variables['lon'][:]
subtitle = "Total between 12:00 AM and %s" % (
lts.strftime("%I:%M %p %Z"),)
routes = 'ac'
if not realtime:
routes = 'a'
# clevs = np.arange(0, 0.25, 0.05)
# clevs = np.append(clevs, np.arange(0.25, 3., 0.25))
# clevs = np.append(clevs, np.arange(3., 10.0, 1))
clevs = [0.01, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8,
10]
sector = 'iowa'
pqstr = ("plot %s %s00 %s_q2_1d.png %s_q2_1d.png png"
) % (routes, ts.strftime("%Y%m%d%H"), sector, sector)
m = MapPlot(title=("%s NCEP MRMS Q3 Today's Precipitation"
) % (ts.strftime("%-d %b %Y"),),
subtitle=subtitle, sector=sector)
(x, y) = np.meshgrid(lons, lats)
m.pcolormesh(x, y, distance(precip, 'MM').value('IN'), clevs,
cmap=nwsprecip(), units='inch')
m.drawcounties()
m.postprocess(pqstr=pqstr, view=False)
m.close()
def snowd(grids, valid, iarchive):
""" Do the snowdepth grid"""
pgconn = psycopg2.connect(database='iem', host='iemdb', user='******')
df = read_sql("""
SELECT ST_x(geom) as lon, ST_y(geom) as lat,
max(snowd) as snow
from summary s JOIN stations t on (s.iemid = t.iemid)
WHERE s.day in (%s, %s) and
t.network in ('IA_COOP', 'MN_COOP', 'WI_COOP', 'IL_COOP',
'MO_COOP', 'NE_COOP', 'KS_COOP', 'SD_COOP') and snowd >= 0
and snowd < 100 GROUP by lon, lat
""", pgconn, params=(valid.date(),
(valid - datetime.timedelta(days=1)).date()),
index_col=None)
nn = NearestNDInterpolator((df['lon'].values, df['lat'].values),
distance(df['snow'].values, 'IN').value('MM'))
grids['snwd'] = nn(XI, YI)
def do(form):
"""Do work!"""
dates = compute_dates(form.getfirst('valid'))
lat = float(form.getfirst("lat"))
lon = float(form.getfirst("lon"))
i, j = prism.find_ij(lon, lat)
res = {'gridi': i, 'gridj': j, 'data': [],
'disclaimer': ("PRISM Climate Group, Oregon State University, "
"http://prism.oregonstate.edu, created 4 Feb 2004.")}
if i is None or j is None:
sys.stdout.write(json.dumps({'error': 'Coordinates outside of domain'}
))
return
for dpair in dates:
sts = dpair[0]
ets = dpair[-1]
sidx = prism.daily_offset(sts)
eidx = prism.daily_offset(ets) + 1
ncfn = "/mesonet/data/prism/%s_daily.nc" % (sts.year, )
nc = netCDF4.Dataset(ncfn, 'r')
tmax = nc.variables['tmax'][sidx:eidx, j, i]
tmin = nc.variables['tmin'][sidx:eidx, j, i]
ppt = nc.variables['ppt'][sidx:eidx, j, i]
nc.close()
for tx, (mt, nt, pt) in enumerate(zip(tmax, tmin, ppt)):
valid = sts + datetime.timedelta(days=tx)
res['data'].append({
'valid': valid.strftime("%Y-%m-%dT12:00:00Z"),
'high_f': myrounder(
datatypes.temperature(mt, 'C').value('F'), 1),
'low_f': myrounder(
datatypes.temperature(nt, 'C').value('F'), 1),
'precip_in': myrounder(
datatypes.distance(pt, 'MM').value('IN'), 2)
})
return json.dumps(res)
def do_work(form):
"""do great things"""
pgconn = get_dbconn('sustainablecorn')
stations = form.getlist('stations')
if not stations:
stations.append("XXX")
sts, ets = get_cgi_dates(form)
df = read_sql("""
SELECT station as uniqueid, valid as day, extract(doy from valid) as doy,
high, low, precip, sknt,
srad_mj, drct from weather_data_daily WHERE station in %s
and valid >= %s and valid <= %s
ORDER by station ASC, valid ASC
""", pgconn, params=(tuple(stations), sts, ets), index_col=None)
df['highc'] = temperature(df['high'].values, 'F').value('C')
df['lowc'] = temperature(df['low'].values, 'F').value('C')
df['precipmm'] = distance(df['precip'].values, 'IN').value('MM')
metarows = [{}, {}]
cols = df.columns
for i, colname in enumerate(cols):
if i == 0:
metarows[0][colname] = 'description'
metarows[1][colname] = 'units'
continue
metarows[0][colname] = VARDF.get(colname, '')
metarows[1][colname] = UVARDF.get(colname, '')
df = pd.concat([pd.DataFrame(metarows), df], ignore_index=True)
# re-establish the correct column sorting
df = df.reindex_axis(cols, axis=1)
writer = pd.ExcelWriter("/tmp/ss.xlsx", engine='xlsxwriter')
df.to_excel(writer, 'Daily Weather', index=False)
worksheet = writer.sheets['Daily Weather']
worksheet.freeze_panes(3, 0)
writer.close()
fn = ",".join(stations)
ssw("Content-type: application/vnd.ms-excel\n")
ssw(("Content-Disposition: attachment;Filename=wx_%s.xls\n\n"
) % (fn, ))
ssw(open('/tmp/ss.xlsx', 'rb').read())
os.unlink('/tmp/ss.xlsx')
def replace_cfs(df, location):
"""Replace the CFS data for this year!"""
pgconn = psycopg2.connect(database='coop', host='iemdb', user='******')
cursor = pgconn.cursor()
coop = XREF[location]['climodat']
today = datetime.date.today() + datetime.timedelta(days=3)
dec31 = today.replace(day=31, month=12)
cursor.execute("""
SELECT day, high, low, precip, srad from alldata_forecast WHERE
modelid = (SELECT id from forecast_inventory WHERE model = 'CFS'
ORDER by modelts DESC LIMIT 1) and station = %s and day >= %s
and day <= %s
""", (coop, today, dec31))
rcols = ['maxt', 'mint', 'rain', 'radn']
for row in cursor:
maxt = temperature(row[1], 'F').value('C')
mint = temperature(row[2], 'F').value('C')
rain = distance(row[3], 'IN').value('MM')
radn = row[4]
df.loc[row[0], rcols] = [maxt, mint, rain, radn]
def doday(ts, realtime):
"""
Create a plot of precipitation stage4 estimates for some day
"""
nc = netCDF4.Dataset("/mesonet/data/iemre/%s_ifc_daily.nc" % (ts.year,))
idx = daily_offset(ts)
xaxis = nc.variables['lon'][:]
yaxis = nc.variables['lat'][:]
total = nc.variables['p01d'][idx, :, :]
nc.close()
lastts = datetime.datetime(ts.year, ts.month, ts.day, 23, 59)
if realtime:
now = datetime.datetime.now() - datetime.timedelta(minutes=60)
lastts = now.replace(minute=59)
subtitle = "Total between 12:00 AM and %s" % (
lastts.strftime("%I:%M %p"),)
routes = 'ac'
if not realtime:
routes = 'a'
clevs = [0.01, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8,
10]
pqstr = ("plot %s %s00 iowa_ifc_1d.png iowa_ifc_1d.png png"
) % (routes, ts.strftime("%Y%m%d%H"))
m = MapPlot(title=("%s Iowa Flood Center Today's Precipitation"
) % (ts.strftime("%-d %b %Y"),),
subtitle=subtitle, sector='custom',
west=xaxis[0], east=xaxis[-1],
south=yaxis[0], north=yaxis[-1])
(x, y) = np.meshgrid(xaxis, yaxis)
m.pcolormesh(x, y, distance(total, 'MM').value("IN"), clevs,
cmap=nwsprecip(), units='inch')
m.drawcounties()
m.postprocess(pqstr=pqstr, view=False)
m.close()
def main():
"""Go Main Go"""
nc = netCDF4.Dataset('/mesonet/data/stage4/2018_stage4_hourly.nc')
precip = nc.variables['p01m']
# Compute needed grid bounds
# y, x
# NW 678, 412
# SW 313, 417
# NE 764, 787
# SE 432, 941
# (west, east, south, north) = compute_bounds(nc)
south = 313
north = 678
west = 412
east = 900
lats = nc.variables['lat'][south:north, west:east]
lons = nc.variables['lon'][south:north, west:east]
pts = []
for lon, lat in zip(np.ravel(lons), np.ravel(lats)):
pts.append(Point(lon, lat))
df = gpd.GeoDataFrame({'geometry': pts})
# iterate over days
now = datetime.date(2018, 7, 1)
ets = datetime.date(2019, 1, 1)
while now < ets:
valid = util.utc(now.year, now.month, now.day, 5)
tidx0 = iemre.hourly_offset(valid)
now += datetime.timedelta(days=1)
valid = util.utc(now.year, now.month, now.day, 5)
tidx1 = iemre.hourly_offset(valid)
data = precip[tidx0:tidx1, south:north, west:east]
total = np.sum(data, axis=0)
print("Processing %s max precip: %.2f" % (
now - datetime.timedelta(days=1), np.max(total)))
df[(now - datetime.timedelta(days=1)).strftime("%b%d")
] = distance(np.ravel(total), 'MM').value("IN")
df.to_file('combined.shp')
def do_output():
pgconn = psycopg2.connect(database='isuag', host='iemdb', user='******')
cursor = pgconn.cursor(cursor_factory=psycopg2.extras.DictCursor)
cursor.execute("""
with data as (
select rank() OVER (ORDER by valid DESC), * from sm_hourly
where valid > now() - '48 hours'::interval)
SELECT *, valid at time zone 'UTC' as utc_valid from data
where rank = 1 ORDER by station ASC""")
sys.stdout.write(("station_id,LAT [deg N],LON [deg E],date_time,ELEV [m],"
"depth [m]#SOILT [C],depth [m]#SOILM [kg/kg]"
"\n"))
nt = NetworkTable("ISUSM")
for row in cursor:
sid = row['station']
sys.stdout.write(
("%s,%.4f,%.4f,%s,%.1f,"
"%.3f;%.3f;%.3f;%.3f#%s;%s;%s;%s,"
"%.3f;%.3f;%.3f#%s;%s;%s,"
"\n"
) % (sid, nt.sts[sid]['lat'],
nt.sts[sid]['lon'],
row['utc_valid'].strftime("%Y-%m-%dT%H:%M:%SZ"),
nt.sts[sid]['elevation'],
distance(4, 'IN').value("M"), distance(12, 'IN').value("M"),
distance(24, 'IN').value("M"), distance(50, 'IN').value("M"),
p(row['tsoil_c_avg'], 3, -90, 90),
p(row['t12_c_avg'], 3, -90, 90),
p(row['t24_c_avg'], 3, -90, 90),
p(row['t50_c_avg'], 3, -90, 90),
distance(12, 'IN').value("M"),
distance(24, 'IN').value("M"), distance(50, 'IN').value("M"),
p(row['vwc_12_avg'], 1, 0, 100),
p(row['vwc_24_avg'], 1, 0, 100),
p(row['vwc_50_avg'], 1, 0, 100),
))
sys.stdout.write(".EOO\n")
def do_month(year, month, routes):
""" Generate a MRMS plot for the month!"""
sts = datetime.datetime(year,month,1)
ets = sts + datetime.timedelta(days=35)
ets = ets.replace(day=1)
today = datetime.datetime.now()
if ets > today:
ets = today
idx0 = iemre.daily_offset(sts)
idx1 = iemre.daily_offset(ets)
nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_mrms_daily.nc" % (year,),
'r')
lats = nc.variables['lat'][:]
lons = nc.variables['lon'][:]
p01d = distance(np.sum(nc.variables['p01d'][idx0:idx1,:,:],0),
'MM').value('IN')
nc.close()
m = MapPlot(sector='iowa', title='MRMS %s - %s Total Precipitation' % (
sts.strftime("%-d %b"),
(ets - datetime.timedelta(days=1)).strftime("%-d %b %Y")),
subtitle='Data from NOAA MRMS Project')
x,y = np.meshgrid(lons, lats)
bins = [0.01, 0.1, 0.5, 1, 1.5, 2, 3, 4, 5, 6, 7, 8, 12, 16, 20]
m.pcolormesh(x, y, p01d, bins, units='inches')
m.drawcounties()
currentfn = "summary/iowa_mrms_q3_month.png"
archivefn = sts.strftime("%Y/%m/summary/iowa_mrms_q3_month.png")
pqstr = "plot %s %s00 %s %s png" % (
routes, sts.strftime("%Y%m%d%H"), currentfn, archivefn)
m.postprocess(pqstr=pqstr)
def process(sheets):
resdf = pd.DataFrame({
'precip': sheets['RainOut']['Rain_mm_Tot'],
'tmpf': sheets['TempRHVPOut']['AirT_C_Avg'],
'rh': sheets['TempRHVPOut']['RH'],
'drct': sheets['WindOut']['WindDir_D1_WVT'],
'sknt': sheets['WindOut']['WS_ms_S_WVT'],
'srad': sheets['SolarRad1Out']['Slr_kW_Avg']})
# Do unit conversion
resdf['srad'] = resdf['srad'] * 1000.
resdf['precip'] = distance(resdf['precip'], 'MM').value('IN')
resdf['tmpf'] = temperature(resdf['tmpf'], 'C').value('F')
resdf['dwpf'] = dewpoint(temperature(resdf['tmpf'], 'F'),
humidity(resdf['rh'], '%')).value('F')
resdf['sknt'] = speed(resdf['sknt'], 'MPS').value('KT')
print(resdf.describe())
minval = resdf.index.min()
maxval = resdf.index.max()
cursor = pgconn.cursor()
cursor.execute("""DELETE from weather_data_obs WHERE
valid between '%s-06' and '%s-06' and station = 'HICKS.P'
""" % (minval.strftime("%Y-%m-%d %H:%M"),
maxval.strftime("%Y-%m-%d %H:%M")))
print("DELETED %s rows between %s and %s" % (cursor.rowcount, minval,
maxval))
for valid, row in resdf.iterrows():
if pd.isnull(valid):
continue
cursor.execute("""INSERT into weather_data_obs
(station, valid, tmpf, dwpf, drct, precip, srad, sknt) VALUES
('HICKS.P', %s, %s, %s, %s, %s, %s, %s)
""", (valid.strftime("%Y-%m-%d %H:%M-06"), row['tmpf'], row['dwpf'],
row['drct'], row['precip'], row['srad'], row['sknt']))
cursor.close()
pgconn.commit()
def do_daily(fn):
df = pd.read_table(fn, sep=' ')
df['sknt'] = speed(df['WINDSPEED'], 'MPS').value('KT')
df['high'] = temperature(df['TMAX'], 'C').value('F')
df['low'] = temperature(df['TMIN'], 'C').value('F')
df['pday'] = distance(df['PRECIP'], 'MM').value('IN')
df['date'] = df[['YEAR', 'MONTH', 'DAY']].apply(lambda x:
datetime.date(x[0], x[1],
x[2]),
axis=1)
print("fn: %s valid: %s - %s" % (fn, df['date'].min(), df['date'].max()))
cursor = pgconn.cursor()
cursor.execute("""DELETE from weather_data_daily where station = 'DPAC'
and valid >= %s and valid <= %s""", (df['date'].min(), df['date'].max()))
if cursor.rowcount > 0:
print("Deleted %s rows" % (cursor.rowcount, ))
for i, row in df.iterrows():
cursor.execute("""INSERT into weather_data_daily
(station, valid, high, low, precip, sknt) VALUES ('DPAC',
%s, %s, %s, %s, %s)""", (row['date'], row['high'], row['low'],
row['pday'], row['sknt']))
print("Inserted %s rows..." % (i + 1, ))
cursor.close()
pgconn.commit()
def do_hourly(fn):
df = pd.read_table(fn, sep=' ')
df['sknt'] = speed(df['WINDSPEED'], 'MPS').value('KT')
df['tmpf'] = temperature(df['TAIR'], 'C').value('F')
df['precip'] = distance(df['PREC'], 'MM').value('IN')
df['valid'] = df[['YEAR', 'MONTH', 'DAY', 'HOUR']].apply(lambda x:
d(*x),
axis=1)
print("fn: %s valid: %s - %s" % (fn, df['valid'].min(),
df['valid'].max()))
cursor = pgconn.cursor()
cursor.execute("""DELETE from weather_data_obs where station = 'DPAC'
and valid >= %s and valid <= %s""", (df['valid'].min(), df['valid'].max()))
if cursor.rowcount > 0:
print("Deleted %s rows" % (cursor.rowcount, ))
for i, row in df.iterrows():
cursor.execute("""INSERT into weather_data_obs
(station, valid, tmpf, sknt, precip, srad) VALUES ('DPAC',
%s, %s, %s, %s, %s)""", (row['valid'].strftime("%Y-%m-%d %H:%M-05"),
row['tmpf'], row['sknt'],
row['precip'], row['RADIATION']))
print("Inserted %s rows..." % (i + 1, ))
cursor.close()
pgconn.commit()
def process(station, metadata):
''' Lets process something, stat
['TMAX', 'TMIN', 'TOBS', 'PRCP', 'SNOW', 'SNWD', 'EVAP', 'MNPN', 'MXPN',
'WDMV', 'DAEV', 'MDEV', 'DAWM', 'MDWM', 'WT05', 'SN01', 'SN02', 'SN03',
'SX01', 'SX02', 'SX03', 'MDPR', 'MDSF', 'SN51', 'SN52', 'SN53', 'SX51',
'SX52', 'SX53', 'WT01', 'SN31', 'SN32', 'SN33', 'SX31', 'SX32', 'SX33']
'''
fp = get_file(station)
if fp is None:
return#
nc = create_netcdf(station, metadata)
if nc is None:
return
data = {}
reclength = len(nc.dimensions['time'])
for line in fp:
m = DATARE.match(line)
d = m.groupdict()
if d['element'] not in ['TMAX', 'TMIN', 'PRCP', 'SNOW', 'SNWD']:
continue
day = datetime.date( int(d['year']), int(d['month']), 1)
days = get_days_for_month(day)
for i in range(1, days+1):
day = day.replace(day=i)
# We don't want data in the future!
if day >= TODAY:
continue
if not data.has_key(day):
data[day] = {}
#if d['flag%s' % (i,)] != " ":
# print d['flag%s' % (i,)]
data[day][d['element']+"mflag"] = d['flag%s' % (i,)][0]
data[day][d['element']+"qflag"] = d['flag%s' % (i,)][1]
data[day][d['element']+"sflag"] = d['flag%s' % (i,)][2]
v = varconv(d['value%s' % (i,)], d['element'])
if v is not None:
data[day][d['element']] = v
table = "alldata_%s" % (station[:2],)
keys = data.keys()
keys.sort()
obs = {}
cursor.execute("""SELECT day, high, low, precip, snow, snowd from
"""+table+""" where station = %s""", (station,))
for row in cursor:
obs[row[0]] = row[1:]
print 'loadvars'
pr_mflag = nc.variables['pr_mflag'][:]
pr_sflag = nc.variables['pr_sflag'][:]
pr_qflag = nc.variables['pr_qflag'][:]
pr = nc.variables['pr'][:]
prsn_mflag = nc.variables['prsn_mflag'][:]
prsn_qflag = nc.variables['prsn_qflag'][:]
prsn_sflag = nc.variables['prsn_sflag'][:]
prsn = nc.variables['prsn'][:]
snowdepth_mflag = nc.variables['snowdepth_mflag'][:]
snowdepth_qflag = nc.variables['snowdepth_qflag'][:]
snowdepth_sflag = nc.variables['snowdepth_sflag'][:]
snowdepth = nc.variables['snowdepth'][:]
tmax_mflag= nc.variables['tmax_mflag'][:]
tmax_qflag= nc.variables['tmax_qflag'][:]
tmax_sflag= nc.variables['tmax_sflag'][:]
tmax= nc.variables['tmax'][:]
tmin_mflag= nc.variables['tmin_mflag'][:]
tmin_qflag= nc.variables['tmin_qflag'][:]
tmin_sflag= nc.variables['tmin_sflag'][:]
tmin= nc.variables['tmin'][:]
#print 'loaded'
for d in keys:
#row = obs.get(d)
offset = (d - BASE).days - 1
pr_mflag[offset] = ord(data[d].get('PRCPmflag', ' '))
pr_qflag[offset] = ord(data[d].get('PRCPqflag', ' '))
pr_sflag[offset] = ord(data[d].get('PRCPsflag', ' '))
if data[d].get('PRCP') is not None:
pr[offset] = distance(data[d].get('PRCP'),
'IN').value("MM") / 86400.
prsn_mflag[offset] = ord(data[d].get('SNOWmflag', ' '))
prsn_qflag[offset] = ord(data[d].get('SNOWqflag', ' '))
prsn_sflag[offset] = ord(data[d].get('SNOWsflag', ' '))
if data[d].get('SNOW') is not None:
prsn[offset] = distance(data[d].get('SNOW'),
'IN').value("MM") / 86400.
snowdepth_mflag[offset] = ord(data[d].get('SNWDmflag', ' '))
snowdepth_qflag[offset] = ord(data[d].get('SNWDqflag', ' '))
snowdepth_sflag[offset] = ord(data[d].get('SNWDsflag', ' '))
if data[d].get('SNWD') is not None:
snowdepth[offset] = distance(data[d].get('SNWD'),
'IN').value("MM")
tmax_mflag[offset] = ord(data[d].get('TMAXmflag', ' '))
tmax_qflag[offset] = ord(data[d].get('TMAXqflag', ' '))
tmax_sflag[offset] = ord(data[d].get('TMAXsflag', ' '))
if data[d].get('TMAX') is not None:
tmax[offset] = temperature(data[d].get('TMAX'),
'F').value('K')
tmin_mflag[offset] = ord(data[d].get('TMINmflag', ' '))
tmin_qflag[offset] = ord(data[d].get('TMINqflag', ' '))
tmin_sflag[offset] = ord(data[d].get('TMINsflag', ' '))
if data[d].get('TMIN') is not None:
tmin[offset] = temperature(data[d].get('TMIN'),
'F').value('K')
if row is None:
print 'No data for %s %s' % (station, d)
cursor.execute("""INSERT into %s(station, day, sday,
year, month) VALUES ('%s', '%s', '%s', %s, %s)""" % (
table, station, d,
"%02i%02i" % (d.month, d.day), d.year,
d.month))
row = [None, None, None, None, None]
s = ""
if (data[d].get('TMAX') is not None and
(row[0] is None or row[0] != data[d]['TMAX'])):
print 'Update %s High %5s -> %5s' % (d, row[0],
data[d]['TMAX'])
s += "high = %.0f," % (data[d]['TMAX'],)
if (data[d].get('TMIN') is not None and
(row[1] is None or row[1] != data[d]['TMIN'])):
print 'Update %s Low %5s -> %5s' % (d, row[1],
data[d]['TMIN'])
s += "low = %.0f," % (data[d]['TMIN'],)
if (data[d].get('PRCP') is not None and
(row[2] is None or row[2] != data[d]['PRCP'])):
print 'Update %s Precip %5s -> %5s' % (d, row[2],
data[d]['PRCP'])
s += "precip = %.2f," % (data[d]['PRCP'],)
if (data[d].get('SNOW') is not None and
(row[3] is None or row[3] != data[d]['SNOW'])):
print 'Update %s Snow %5s -> %5s' % (d, row[3],
data[d]['SNOW'])
s += "snow = %.1f," % (data[d]['SNOW'],)
if (data[d].get('SNWD') is not None and
(row[4] is None or row[4] != data[d]['SNWD'])):
print 'Update %s Snowd %5s -> %5s' % (d, row[4],
data[d]['SNWD'])
s += "snowd = %.1f," % (data[d]['SNWD'],)
if s != "":
sql = """UPDATE %s SET %s WHERE day = '%s' and
station = '%s'""" % (table, s[:-1],
d, station )
cursor.execute(sql)
#print 'Write pr netcdf'
nc.variables['pr'][:] = pr
#print 'Write pr_mflag netcdf'
nc.variables['pr_mflag'][:] = pr_mflag
#print 'Write pr_qflag netcdf'
nc.variables['pr_qflag'][:] = pr_qflag
#print 'Write pr_sflag netcdf'
nc.variables['pr_sflag'][:] = pr_sflag
#print 'Write prsn netcdf'
nc.variables['prsn'][:] = prsn
nc.variables['prsn_mflag'][:] = prsn_mflag
nc.variables['prsn_qflag'][:] = prsn_qflag
nc.variables['prsn_sflag'][:] = prsn_sflag
#print 'Write snowdepth netcdf'
nc.variables['snowdepth'][:] = snowdepth
nc.variables['snowdepth_mflag'][:] = snowdepth_mflag
nc.variables['snowdepth_qflag'][:] = snowdepth_qflag
nc.variables['snowdepth_sflag'][:] = snowdepth_sflag
#print 'Write tmax netcdf'
nc.variables['tmax'][:] = tmax
nc.variables['tmax_mflag'][:] = tmax_mflag
nc.variables['tmax_qflag'][:] = tmax_qflag
nc.variables['tmax_sflag'][:] = tmax_sflag
#print 'Write tmin netcdf'
nc.variables['tmin'][:] = tmin
nc.variables['tmin_mflag'][:] = tmin_mflag
nc.variables['tmin_qflag'][:] = tmin_qflag
nc.variables['tmin_sflag'][:] = tmin_sflag
#print 'Done writing'
nc.close()
basefn = compute_fn(info['lon'], info['lat'])
# print basefn, info['lon'], info['lat']
# load precip
fn = "../arrm/swatfiles_%s/%s.pcp" % (model, basefn)
if not os.path.isfile(fn):
continue
obs = {}
for line in open(fn):
if not line.startswith('20'):
continue
if line.find('-99.0') > 0:
line = line.replace('-99.0', ' 0.0')
valid = datetime.datetime.strptime(line[:7], '%Y%j')
obs[valid] = {'valid': valid,
'precip_in':
distance(float(line[7:12]), 'MM').value('IN')}
fn = "../arrm/swatfiles_%s/%s.tmp" % (model, basefn)
for line in open(fn):
if not line.startswith('20'):
continue
if line.find('-99.0') > 0:
continue
valid = datetime.datetime.strptime(line[:7], '%Y%j')
obs[valid]['high_f'] = temperature(float(line[7:12]), 'C').value('F')
obs[valid]['low_f'] = temperature(float(line[12:17]), 'C').value('F')
res = {'fips': fips}
rows = []
for key, row in obs.iteritems():
rows.append(row)
df = pd.DataFrame(rows)
import pygrib
from pyiem.plot import MapPlot
from pyiem import reference
import numpy as np
from pyiem.datatypes import distance
g = pygrib.open('p168m_2016090600f168.grb')
grb = g[1]
lats, lons = grb.latlons()
m = MapPlot(sector='custom', project='aea',
south=reference.MW_SOUTH, north=reference.MW_NORTH,
east=reference.MW_EAST, west=reference.MW_WEST,
axisbg='tan',
title=("Weather Prediction Center (WPC) "
"7 Day Forecasted Precipitation"),
subtitle='7 PM 5 September thru 7 PM 12 September 2016')
m.contourf(lons, lats, distance(grb['values'], 'MM').value('IN'),
np.arange(0, 4.6, 0.25), units='inches')
m.postprocess(filename='test.png')