def replace_cfs(nc, valid, islice, jslice):
"""Copy CFS data into the given year."""
tidx0 = (valid - datetime.date(valid.year, 1, 1)).days
tidx1 = (
datetime.date(valid.year, 12, 31) - datetime.date(valid.year, 1, 1)
).days
cfsnc = ncopen(valid.strftime("/mesonet/data/iemre/cfs_%Y%m%d%H.nc"))
tidx = iemre.daily_offset(valid + datetime.timedelta(days=1))
tslice = slice(tidx0 + 1, tidx1 + 1)
# print("replace_cfs filling %s from %s" % (tslice, tidx))
# CFS is W m-2, we want MJ
nc.variables["srad"][tslice, :, :] = (
cfsnc.variables["srad"][tidx:, jslice, islice] * 86400.0 / 1000000.0
)
highc = temperature(
cfsnc.variables["high_tmpk"][tidx:, jslice, islice], "K"
).value("C")
lowc = temperature(
cfsnc.variables["low_tmpk"][tidx:, jslice, islice], "K"
).value("C")
nc.variables["tmax"][tslice, :, :] = highc
nc.variables["tmin"][tslice, :, :] = lowc
nc.variables["gdd_f"][tslice, :, :] = gdd(
temperature(highc, "C"), temperature(lowc, "C")
)
nc.variables["prcp"][tslice, :, :] = cfsnc.variables["p01d"][
tidx:, jslice, islice
]
cfsnc.close()
def main():
"""Go Main Go"""
os.chdir("baseline")
for fn in glob.glob("*.met"):
location = fn[:-4]
output = open("%s.csv" % (location, ), "w")
output.write("date,high[F],low[F],precip[inch],gdd[F]\n")
for line in open(fn):
line = line.strip()
if (not line.startswith("2012") and not line.startswith("2015")
and not line.startswith("2016")):
continue
tokens = line.split()
valid = datetime.date(int(
tokens[0]), 1, 1) + datetime.timedelta(days=int(tokens[1]) - 1)
high = temperature(float(tokens[3]), "C")
low = temperature(float(tokens[4]), "C")
gdd = met.gdd(high, low, 50, 86)
precip = distance(float(tokens[5]), "MM")
output.write(("%s,%.1f,%.1f,%.2f,%.1f\n") % (
valid.strftime("%Y-%m-%d"),
high.value("F"),
low.value("F"),
precip.value("IN"),
gdd,
))
output.close()
def load(dirname, location, sdate):
""" Read a file please """
data = []
idx = []
mindoy = int(sdate.strftime("%j"))
for line in open("%s/%s.met" % (dirname, location)):
line = line.strip()
if not line.startswith('19') and not line.startswith('20'):
continue
tokens = line.split()
if int(tokens[1]) < mindoy:
continue
data.append(tokens)
ts = (datetime.date(int(tokens[0]), 1, 1) +
datetime.timedelta(days=int(tokens[1])-1))
idx.append(ts)
if len(data[0]) < 10:
cols = ['year', 'doy', 'radn', 'maxt', 'mint', 'rain']
else:
cols = ['year', 'doy', 'radn', 'maxt', 'mint',
'rain', 'gdd', 'st4', 'st12', 'st24',
'st50', 'sm12', 'sm24', 'sm50']
df = pd.DataFrame(data, index=idx,
columns=cols)
for col in cols:
df[col] = pd.to_numeric(df[col], errors='coerce')
if len(data[0]) < 10:
df['gdd'] = gdd(temperature(df['maxt'].values, 'C'),
temperature(df['mint'].values, 'C'))
df['gddcum'] = df.groupby(['year'])['gdd'].apply(lambda x: x.cumsum())
df['raincum'] = distance(
df.groupby(['year'])['rain'].apply(lambda x: x.cumsum()),
'MM').value('IN')
return df
def test_gdd_with_nans():
"""Can we properly deal with nan's and not emit warnings?"""
highs = np.ma.array([70, 80, np.nan, 90], mask=[False, False, True, False])
lows = highs - 10
r = meteorology.gdd(datatypes.temperature(highs, 'F'),
datatypes.temperature(lows, 'F'), 50, 86)
assert np.ma.is_masked(r[2])
def load(dirname, location, sdate):
""" Read a file please """
data = []
idx = []
mindoy = int(sdate.strftime("%j"))
for line in open("%s/%s.met" % (dirname, location)):
line = line.strip()
if not line.startswith('19') and not line.startswith('20'):
continue
tokens = line.split()
if int(tokens[1]) < mindoy:
continue
data.append(tokens)
ts = (datetime.date(int(tokens[0]), 1, 1) +
datetime.timedelta(days=int(tokens[1])-1))
idx.append(ts)
if len(data[0]) < 10:
cols = ['year', 'doy', 'radn', 'maxt', 'mint', 'rain']
else:
cols = ['year', 'doy', 'radn', 'maxt', 'mint',
'rain', 'gdd', 'st4', 'st12', 'st24',
'st50', 'sm12', 'sm24', 'sm50']
df = pd.DataFrame(data, index=idx,
columns=cols)
for col in cols:
df[col] = pd.to_numeric(df[col], errors='coerce')
if len(data[0]) < 10:
df['gdd'] = gdd(temperature(df['maxt'].values, 'C'),
temperature(df['mint'].values, 'C'))
df['gddcum'] = df.groupby(['year'])['gdd'].apply(lambda x: x.cumsum())
df['raincum'] = distance(
df.groupby(['year'])['rain'].apply(lambda x: x.cumsum()),
'MM').value('IN')
return df
def replace_obs_iem(df, location):
"""Replace dataframe data with obs for this location
Tricky part, if the baseline already provides data for this year, we should
use it!
"""
pgconn = get_dbconn("iem", user="******")
cursor = pgconn.cursor()
station = XREF[location]["station"]
today = datetime.date.today()
jan1 = today.replace(month=1, day=1)
years = [
int(y) for y in np.arange(df.index.values.min().year,
df.index.values.max().year + 1)
]
table = "summary_%s" % (jan1.year, )
cursor.execute(
"""
select day, max_tmpf, min_tmpf, srad_mj, pday
from """ + table + """ s JOIN stations t on (s.iemid = t.iemid)
WHERE t.id = %s and max_tmpf is not null
and day < 'TODAY' ORDER by day ASC
""",
(station, ),
)
rcols = ["maxt", "mint", "radn", "gdd", "rain"]
replaced = []
for row in cursor:
valid = row[0]
# Does our df currently have data for this date? If so, we shall do
# no more
dont_replace = not np.isnan(df.at[valid, "mint"])
if not dont_replace:
replaced.append(valid)
_gdd = gdd(temperature(row[1], "F"), temperature(row[2], "F"))
for year in years:
if valid.month == 2 and valid.day == 29 and year % 4 != 0:
continue
if dont_replace:
df.loc[valid.replace(year=year), rcols[3:]] = (
_gdd,
distance(row[4], "in").value("mm"),
)
continue
df.loc[valid.replace(year=year), rcols] = (
temperature(row[1], "F").value("C"),
temperature(row[2], "F").value("C"),
row[3],
_gdd,
distance(row[4], "in").value("mm"),
)
if replaced:
LOG.info(
" used IEM Access %s from %s->%s",
station,
replaced[0],
replaced[-1],
)
def test_gdd():
"""Growing Degree Days"""
r = meteorology.gdd(datatypes.temperature(86, 'F'),
datatypes.temperature(50, 'F'), 50, 86)
assert r == 18
r = meteorology.gdd(datatypes.temperature(51, 'F'),
datatypes.temperature(49, 'F'), 50, 86)
assert abs(r - 0.5) < 0.1
r = meteorology.gdd(datatypes.temperature(49, 'F'),
datatypes.temperature(40, 'F'), 50, 86)
assert r == 0
r = meteorology.gdd(datatypes.temperature([86, 86], 'F'),
datatypes.temperature([50, 50], 'F'), 50, 86)
assert r[0] == 18
assert r[1] == 18
def test_gdd(self):
"""Growing Degree Days"""
r = meteorology.gdd(datatypes.temperature(86, 'F'),
datatypes.temperature(50, 'F'), 50, 86)
self.assertEquals(r, 18)
r = meteorology.gdd(datatypes.temperature(51, 'F'),
datatypes.temperature(49, 'F'), 50, 86)
self.assertAlmostEquals(r, 0.5, 1)
r = meteorology.gdd(datatypes.temperature(49, 'F'),
datatypes.temperature(40, 'F'), 50, 86)
self.assertEquals(r, 0)
r = meteorology.gdd(datatypes.temperature([86, 86], 'F'),
datatypes.temperature([50, 50], 'F'), 50, 86)
self.assertEquals(r[0], 18)
self.assertEquals(r[1], 18)
def test_gdd_with_nans():
"""Can we properly deal with nan's and not emit warnings?"""
highs = np.ma.array([70, 80, np.nan, 90],
mask=[False, False, True, False])
lows = highs - 10
r = meteorology.gdd(datatypes.temperature(highs, 'F'),
datatypes.temperature(lows, 'F'),
50, 86)
assert np.ma.is_masked(r[2])
def load(dirname, location, sdate):
""" Read a file please """
data = []
idx = []
for line in open("%s/%s.met" % (dirname, location)):
line = line.strip()
if not line.startswith("19") and not line.startswith("20"):
continue
tokens = line.split()
data.append(tokens)
ts = datetime.date(int(tokens[0]), 1, 1) + datetime.timedelta(days=int(tokens[1]) - 1)
idx.append(ts)
if len(data[0]) < 10:
cols = ["year", "doy", "radn", "maxt", "mint", "rain"]
else:
cols = [
"year",
"doy",
"radn",
"maxt",
"mint",
"rain",
"gdd",
"st4",
"st12",
"st24",
"st50",
"sm12",
"sm24",
"sm50",
]
df = pd.DataFrame(data, index=idx, columns=cols)
for col in cols:
df[col] = pd.to_numeric(df[col], errors="coerce")
if len(data[0]) < 10:
df["gdd"] = gdd(temperature(df["maxt"].values, "C"), temperature(df["mint"].values, "C"))
bins = []
today = datetime.date.today()
for valid, _ in df.iterrows():
if valid >= today:
bins.append(0)
continue
if sdate == "nov1" and valid.month >= 11:
bins.append(valid.year + 1)
continue
if valid.month < today.month:
bins.append(valid.year)
continue
if valid.month == today.month and valid.day < today.day:
bins.append(valid.year)
continue
bins.append(0)
df["bin"] = bins
df["rain"] = distance(df["rain"].values, "MM").value("IN")
df["avgt"] = temperature((df["maxt"] + df["mint"]) / 2.0, "C").value("F")
return df
def copy_iemre(nc, fromyear, ncdate0, ncdate1, islice, jslice):
"""Copy IEMRE data from a given year to **inclusive** dates."""
rencfn = iemre.get_daily_ncname(fromyear)
if not os.path.isfile(rencfn):
print("reanalysis fn %s missing" % (rencfn,))
return
renc = ncopen(rencfn)
tidx0 = (ncdate0 - datetime.date(fromyear, 1, 1)).days
tidx1 = (ncdate1 - datetime.date(fromyear, 1, 1)).days
tslice = slice(tidx0, tidx1 + 1)
# time steps to fill
tsteps = (tidx1 - tidx0) + 1
# figure out the slice
if ncdate0.strftime("%m%d") == "0101":
retslice = slice(0, tsteps)
else:
retslice = slice(0 - tsteps, None)
# print("copy_iemre from %s filling %s steps nc: %s iemre: %s" % (
# fromyear, tsteps, tslice, retslice
# ))
highc = temperature(
renc.variables["high_tmpk"][retslice, jslice, islice], "K"
).value("C")
lowc = temperature(
renc.variables["low_tmpk"][retslice, jslice, islice], "K"
).value("C")
nc.variables["tmax"][tslice, :, :] = highc
nc.variables["tmin"][tslice, :, :] = lowc
nc.variables["gdd_f"][tslice, :, :] = gdd(
temperature(highc, "C"), temperature(lowc, "C")
)
nc.variables["prcp"][tslice, :, :] = renc.variables["p01d"][
retslice, jslice, islice
]
for rt, nt in zip(
list(
range(
retslice.start, 0 if retslice.stop is None else retslice.stop
)
),
list(range(tslice.start, tslice.stop)),
):
# IEMRE power_swdn is MJ, test to see if data exists
srad = renc.variables["power_swdn"][rt, jslice, islice]
if srad.mask.any():
# IEMRE rsds uses W m-2, we want MJ
srad = (
renc.variables["rsds"][rt, jslice, islice]
* 86400.0
/ 1000000.0
)
nc.variables["srad"][nt, :, :] = srad
renc.close()
def replace_obs(df, location):
"""Replace dataframe data with obs for this location
Tricky part, if the baseline already provides data for this year, we should
use it!
"""
pgconn = get_dbconn('isuag', user='******')
cursor = pgconn.cursor()
isusm = XREF[location]['isusm']
today = datetime.date.today()
jan1 = today.replace(month=1, day=1)
years = [
int(y) for y in np.arange(df.index.values.min().year,
df.index.values.max().year + 1)
]
cursor.execute(
"""
select valid, tair_c_max_qc, tair_c_min_qc, slrmj_tot_qc,
vwc_12_avg_qc,
vwc_24_avg_qc, vwc_50_avg_qc, tsoil_c_avg_qc, t12_c_avg_qc,
t24_c_avg_qc, t50_c_avg_qc,
rain_mm_tot_qc from sm_daily WHERE station = %s and valid >= %s
and tair_c_max_qc is not null and tair_c_min_qc is not null
ORDER by valid
""", (isusm, jan1))
rcols = [
'maxt', 'mint', 'radn', 'gdd', 'sm12', 'sm24', 'sm50', 'st4', 'st12',
'st24', 'st50', 'rain'
]
replaced = []
for row in cursor:
valid = row[0]
# Does our df currently have data for this date? If so, we shall do
# no more
dont_replace = not np.isnan(df.at[valid, 'mint'])
if not dont_replace:
replaced.append(valid)
_gdd = gdd(temperature(row[1], 'C'), temperature(row[2], 'C'))
for year in years:
if valid.month == 2 and valid.day == 29 and year % 4 != 0:
continue
if dont_replace:
df.loc[valid.replace(year=year),
rcols[3:-1]] = (_gdd, row[4], row[5], row[6], row[7],
row[8], row[9], row[10])
continue
df.loc[valid.replace(year=year),
rcols] = (row[1], row[2], row[3], _gdd, row[4], row[5],
row[6], row[7], row[8], row[9], row[10], row[11])
if replaced:
print((" replaced with obs from %s for %s->%s") %
(isusm, replaced[0], replaced[-1]))
def load(dirname, location, sdate):
""" Read a file please """
data = []
idx = []
mindoy = int(sdate.strftime("%j"))
fn = "%s/%s.met" % (dirname, location)
if not os.path.isfile(fn):
raise NoDataFound("Data file was not found.")
for line in open(fn):
line = line.strip()
if not line.startswith("19") and not line.startswith("20"):
continue
tokens = line.split()
if int(tokens[1]) < mindoy:
continue
data.append(tokens)
ts = datetime.date(int(tokens[0]), 1, 1) + datetime.timedelta(
days=int(tokens[1]) - 1
)
idx.append(ts)
if len(data[0]) < 10:
cols = ["year", "doy", "radn", "maxt", "mint", "rain"]
else:
cols = [
"year",
"doy",
"radn",
"maxt",
"mint",
"rain",
"gdd",
"st4",
"st12",
"st24",
"st50",
"sm12",
"sm24",
"sm50",
]
df = pd.DataFrame(data, index=idx, columns=cols)
for col in cols:
df[col] = pd.to_numeric(df[col], errors="coerce")
if len(data[0]) < 10:
df["gdd"] = gdd(
temperature(df["maxt"].values, "C"),
temperature(df["mint"].values, "C"),
)
df["gddcum"] = df.groupby(["year"])["gdd"].apply(lambda x: x.cumsum())
df["raincum"] = distance(
df.groupby(["year"])["rain"].apply(lambda x: x.cumsum()), "MM"
).value("IN")
return df
def test_gdd():
"""Growing Degree Days"""
r = meteorology.gdd(datatypes.temperature(86, 'F'),
datatypes.temperature(50, 'F'),
50, 86)
assert r == 18
r = meteorology.gdd(datatypes.temperature(51, 'F'),
datatypes.temperature(49, 'F'),
50, 86)
assert abs(r - 0.5) < 0.1
r = meteorology.gdd(datatypes.temperature(49, 'F'),
datatypes.temperature(40, 'F'),
50, 86)
assert r == 0
r = meteorology.gdd(datatypes.temperature([86, 86], 'F'),
datatypes.temperature([50, 50], 'F'),
50, 86)
assert r[0] == 18
assert r[1] == 18
def test_gdd(self):
"""Growing Degree Days"""
r = meteorology.gdd(datatypes.temperature(86, 'F'),
datatypes.temperature(50, 'F'),
50, 86)
self.assertEquals(r, 18)
r = meteorology.gdd(datatypes.temperature(51, 'F'),
datatypes.temperature(49, 'F'),
50, 86)
self.assertAlmostEquals(r, 0.5, 1)
r = meteorology.gdd(datatypes.temperature(49, 'F'),
datatypes.temperature(40, 'F'),
50, 86)
self.assertEquals(r, 0)
r = meteorology.gdd(datatypes.temperature([86, 86], 'F'),
datatypes.temperature([50, 50], 'F'),
50, 86)
self.assertEquals(r[0], 18)
self.assertEquals(r[1], 18)
def load(dirname, location, sdate):
""" Read a file please """
data = []
idx = []
fn = "%s/%s.met" % (dirname, location)
if not os.path.isfile(fn):
raise NoDataFound("File was not found.")
for line in open(fn):
line = line.strip()
if not line.startswith('19') and not line.startswith('20'):
continue
tokens = line.split()
if float(tokens[5]) > 90:
continue
data.append(tokens)
ts = (datetime.date(int(tokens[0]), 1, 1) +
datetime.timedelta(days=int(tokens[1]) - 1))
idx.append(ts)
if len(data[0]) < 10:
cols = ['year', 'doy', 'radn', 'maxt', 'mint', 'rain']
else:
cols = [
'year', 'doy', 'radn', 'maxt', 'mint', 'rain', 'gdd', 'st4',
'st12', 'st24', 'st50', 'sm12', 'sm24', 'sm50'
]
df = pd.DataFrame(data, index=idx, columns=cols)
for col in cols:
df[col] = pd.to_numeric(df[col], errors='coerce')
if len(data[0]) < 10:
df['gdd'] = gdd(temperature(df['maxt'].values, 'C'),
temperature(df['mint'].values, 'C'))
bins = []
today = datetime.date.today()
for valid, _ in df.iterrows():
if valid >= today:
bins.append(0)
continue
if sdate == 'nov1' and valid.month >= 11:
bins.append(valid.year + 1)
continue
if valid.month < today.month:
bins.append(valid.year)
continue
if valid.month == today.month and valid.day < today.day:
bins.append(valid.year)
continue
bins.append(0)
df['bin'] = bins
df['rain'] = distance(df['rain'].values, 'MM').value('IN')
df['avgt'] = temperature((df['maxt'] + df['mint']) / 2.0, 'C').value('F')
return df
def replace_obs_iem(df, location):
"""Replace dataframe data with obs for this location
Tricky part, if the baseline already provides data for this year, we should
use it!
"""
pgconn = psycopg2.connect(database='iem', host='iemdb', user='******')
cursor = pgconn.cursor()
station = XREF[location]['station']
today = datetime.date.today()
jan1 = today.replace(month=1, day=1)
years = [int(y) for y in np.arange(df.index.values.min().year,
df.index.values.max().year + 1)]
table = "summary_%s" % (jan1.year,)
cursor.execute("""
select day, max_tmpf, min_tmpf, srad_mj, pday
from """ + table + """ s JOIN stations t on (s.iemid = t.iemid)
WHERE t.id = %s and max_tmpf is not null
and day < 'TODAY' ORDER by day ASC
""", (station,))
rcols = ['maxt', 'mint', 'radn', 'gdd', 'rain']
replaced = []
for row in cursor:
valid = row[0]
# Does our df currently have data for this date? If so, we shall do
# no more
dont_replace = not np.isnan(df.at[valid, 'mint'])
if not dont_replace:
replaced.append(valid)
_gdd = gdd(temperature(row[1], 'F'), temperature(row[2], 'F'))
for year in years:
if valid.month == 2 and valid.day == 29 and year % 4 != 0:
continue
if dont_replace:
df.loc[valid.replace(year=year),
rcols[3:-1]] = (_gdd,
distance(row[4], 'in').value('mm'))
continue
df.loc[valid.replace(year=year), rcols] = (
temperature(row[1], 'F').value('C'),
temperature(row[2], 'F').value('C'), row[3],
_gdd, distance(row[4], 'in').value('mm'))
if len(replaced) > 0:
print((" used IEM Access %s from %s->%s"
) % (station, replaced[0], replaced[-1]))
def plot_gdd(ts):
"""Generate our plot."""
nc = ncopen(ts.strftime("/mesonet/data/ndfd/%Y%m%d%H_ndfd.nc"))
# compute our daily GDDs
gddtot = np.zeros(np.shape(nc.variables["lon"][:]))
for i in range(7):
gddtot += gdd(
temperature(nc.variables["high_tmpk"][i, :, :], "K"),
temperature(nc.variables["low_tmpk"][i, :, :], "K"),
)
cnc = ncopen("/mesonet/data/ndfd/ndfd_dailyc.nc")
offset = daily_offset(ts)
avggdd = np.sum(cnc.variables["gdd50"][offset:offset + 7], 0)
data = gddtot - np.where(avggdd < 1, 1, avggdd)
subtitle = ("Based on National Digital Forecast Database (NDFD) "
"00 UTC Forecast made %s") % (ts.strftime("%-d %b %Y"), )
mp = MapPlot(
title="NWS NDFD 7 Day (%s through %s) GDD50 Departure from Avg" % (
ts.strftime("%-d %b"),
(ts + datetime.timedelta(days=6)).strftime("%-d %b"),
),
subtitle=subtitle,
sector="iailin",
)
mp.pcolormesh(
nc.variables["lon"][:],
nc.variables["lat"][:],
data,
np.arange(-80, 81, 20),
cmap=plt.get_cmap("RdBu_r"),
units=r"$^\circ$F",
spacing="proportional",
)
mp.drawcounties()
pqstr = (
"data c %s summary/cb_ndfd_7day_gdd.png summary/cb_ndfd_7day_gdd.png "
"png") % (ts.strftime("%Y%m%d%H%M"), )
mp.postprocess(pqstr=pqstr)
mp.close()
nc.close()
def copy_iemre(nc, ncdate0, ncdate1, islice, jslice):
"""Copy IEMRE data from a given year to **inclusive** dates."""
rencfn = iemre.get_daily_ncname(ncdate0.year)
if not os.path.isfile(rencfn):
LOG.info("reanalysis fn %s missing", rencfn)
return
with ncopen(rencfn) as renc:
# Compute offsets for yieldfx file
tidx0 = (ncdate0 - datetime.date(1980, 1, 1)).days
tidx1 = (ncdate1 - datetime.date(1980, 1, 1)).days
yfx_slice = slice(tidx0, tidx1 + 1)
# Compute offsets for the reanalysis file
tidx0 = (ncdate0 - datetime.date(ncdate0.year, 1, 1)).days
tidx1 = (ncdate1 - datetime.date(ncdate0.year, 1, 1)).days
re_slice = slice(tidx0, tidx1 + 1)
# LOG.debug("filling nc: %s iemre: %s", yfx_slice, re_slice)
highc = temperature(
renc.variables["high_tmpk"][re_slice, jslice, islice],
"K").value("C")
lowc = temperature(
renc.variables["low_tmpk"][re_slice, jslice, islice],
"K").value("C")
nc.variables["tmax"][yfx_slice, :, :] = highc
nc.variables["tmin"][yfx_slice, :, :] = lowc
nc.variables["gdd_f"][yfx_slice, :, :] = gdd(temperature(highc, "C"),
temperature(lowc, "C"))
nc.variables["prcp"][yfx_slice, :, :] = renc.variables["p01d"][
re_slice, jslice, islice]
# Special care needed for solar radiation filling
for rt, nt in zip(
list(range(re_slice.start, re_slice.stop)),
list(range(yfx_slice.start, yfx_slice.stop)),
):
# IEMRE power_swdn is MJ, test to see if data exists
srad = renc.variables["power_swdn"][rt, jslice, islice]
if srad.mask.any():
# IEMRE rsds uses W m-2, we want MJ
srad = (renc.variables["rsds"][rt, jslice, islice] * 86400.0 /
1000000.0)
nc.variables["srad"][nt, :, :] = srad
def compute_gdd(df):
"""Compute GDDs Please"""
df['gdd'] = gdd(temperature(df['maxt'].values, 'C'),
temperature(df['mint'].values, 'C'))
def replace_obs(df, location):
"""Replace dataframe data with obs for this location
Tricky part, if the baseline already provides data for this year, we should
use it!
"""
pgconn = psycopg2.connect(database='isuag', host='iemdb', user='******')
cursor = pgconn.cursor()
isusm = XREF[location]['isusm']
today = datetime.date.today()
jan1 = today.replace(month=1, day=1)
years = [int(y) for y in np.arange(df.index.values.min().year,
df.index.values.max().year + 1)]
cursor.execute("""
select valid, tair_c_max_qc, tair_c_min_qc, slrmj_tot_qc,
vwc_12_avg_qc,
vwc_24_avg_qc, vwc_50_avg_qc, tsoil_c_avg_qc, t12_c_avg_qc,
t24_c_avg_qc, t50_c_avg_qc,
rain_mm_tot_qc from sm_daily WHERE station = %s and valid >= %s
ORDER by valid
""", (isusm, jan1))
rcols = ['maxt', 'mint', 'radn', 'gdd', 'sm12', 'sm24', 'sm50',
'st4', 'st12', 'st24', 'st50', 'rain']
replaced = []
for row in cursor:
valid = row[0]
# Does our df currently have data for this date? If so, we shall do
# no more
dont_replace = not np.isnan(df.at[valid, 'mint'])
if not dont_replace:
replaced.append(valid)
_gdd = gdd(temperature(row[1], 'C'), temperature(row[2], 'C'))
for year in years:
if valid.month == 2 and valid.day == 29 and year % 4 != 0:
continue
if dont_replace:
df.loc[valid.replace(year=year),
rcols[3:-1]] = (_gdd, row[4], row[5],
row[6], row[7], row[8],
row[9], row[10])
continue
df.loc[valid.replace(year=year), rcols] = (row[1], row[2], row[3],
_gdd, row[4], row[5],
row[6], row[7], row[8],
row[9], row[10], row[11]
)
if len(replaced) > 0:
print("yieldfx_workflow supplemented %s from %s to %s" % (location,
replaced[0],
replaced[-1]
))
# We no longer want to use rain from climodat...
return
# Go get precip from Climodat
coop = XREF[location]['climodat']
pgconn = psycopg2.connect(database='coop', host='iemdb', user='******')
cursor = pgconn.cursor()
cursor.execute("""
SELECT day, precip from alldata_ia where year = %s and station = %s
and day < %s
""", (jan1.year, coop, today))
for row in cursor:
valid = row[0]
pcpn = distance(row[1], 'IN').value('MM')
for year in years:
if valid.month == 2 and valid.day == 29 and year % 4 != 0:
continue
df.at[valid.replace(year=year), 'rain'] = pcpn
"""Dump baseline file as CSV, with GDD computed"""
from pyiem.datatypes import temperature, distance
import pyiem.meteorology as met
import glob
import os
import datetime
os.chdir('baseline')
for fn in glob.glob("*.met"):
location = fn[:-4]
o = open("%s.csv" % (location,), 'w')
o.write("date,high[F],low[F],precip[inch],gdd[F]\n")
for line in open(fn):
line = line.strip()
if not line.startswith('2012') and not line.startswith('2015'):
continue
tokens = line.split()
valid = (datetime.date(int(tokens[0]), 1, 1) +
datetime.timedelta(days=int(tokens[1])-1))
high = temperature(float(tokens[3]), 'C')
low = temperature(float(tokens[4]), 'C')
gdd = met.gdd(high, low, 50, 86)
precip = distance(float(tokens[5]), 'MM')
o.write(("%s,%.1f,%.1f,%.2f,%.1f\n"
) % (valid.strftime("%Y-%m-%d"), high.value("F"),
low.value("F"), precip.value("IN"), gdd))
o.close()
def compute_gdd(df):
"""Compute GDDs Please"""
df['gdd'] = gdd(temperature(df['maxt'], 'C'), temperature(df['mint'], 'C'))
def load(dirname, location, sdate):
""" Read a file please """
data = []
idx = []
fn = "%s/%s.met" % (dirname, location)
if not os.path.isfile(fn):
raise NoDataFound("File was not found.")
for line in open(fn):
line = line.strip()
if not line.startswith("19") and not line.startswith("20"):
continue
tokens = line.split()
if float(tokens[5]) > 90:
continue
data.append(tokens)
ts = datetime.date(int(tokens[0]), 1,
1) + datetime.timedelta(days=int(tokens[1]) - 1)
idx.append(ts)
if len(data[0]) < 10:
cols = ["year", "doy", "radn", "maxt", "mint", "rain"]
else:
cols = [
"year",
"doy",
"radn",
"maxt",
"mint",
"rain",
"gdd",
"st4",
"st12",
"st24",
"st50",
"sm12",
"sm24",
"sm50",
]
df = pd.DataFrame(data, index=idx, columns=cols)
for col in cols:
df[col] = pd.to_numeric(df[col], errors="coerce")
if len(data[0]) < 10:
df["gdd"] = gdd(
temperature(df["maxt"].values, "C"),
temperature(df["mint"].values, "C"),
)
bins = []
today = datetime.date.today()
for valid, _ in df.iterrows():
if valid >= today:
bins.append(0)
continue
if sdate == "nov1" and valid.month >= 11:
bins.append(valid.year + 1)
continue
if valid.month < today.month:
bins.append(valid.year)
continue
if valid.month == today.month and valid.day < today.day:
bins.append(valid.year)
continue
bins.append(0)
df["bin"] = bins
df["rain"] = distance(df["rain"].values, "MM").value("IN")
df["avgt"] = temperature((df["maxt"] + df["mint"]) / 2.0, "C").value("F")
return df
def compute_gdd(df):
"""Compute GDDs Please"""
df["gdd"] = gdd(
temperature(df["maxt"].values, "C"),
temperature(df["mint"].values, "C"),
)
def replace_obs(df, location):
"""Replace dataframe data with obs for this location
Tricky part, if the baseline already provides data for this year, we should
use it!
"""
pgconn = psycopg2.connect(database='isuag', host='iemdb', user='******')
cursor = pgconn.cursor()
isusm = XREF[location]['isusm']
today = datetime.date.today()
jan1 = today.replace(month=1, day=1)
years = [
int(y) for y in np.arange(df.index.values.min().year,
df.index.values.max().year + 1)
]
cursor.execute(
"""
select valid, tair_c_max_qc, tair_c_min_qc, slrmj_tot_qc,
vwc_12_avg_qc,
vwc_24_avg_qc, vwc_50_avg_qc, tsoil_c_avg_qc, t12_c_avg_qc,
t24_c_avg_qc, t50_c_avg_qc,
rain_mm_tot_qc from sm_daily WHERE station = %s and valid >= %s
ORDER by valid
""", (isusm, jan1))
rcols = [
'maxt', 'mint', 'radn', 'gdd', 'sm12', 'sm24', 'sm50', 'st4', 'st12',
'st24', 'st50', 'rain'
]
replaced = []
for row in cursor:
valid = row[0]
# Does our df currently have data for this date? If so, we shall do
# no more
dont_replace = not np.isnan(df.at[valid, 'mint'])
if not dont_replace:
replaced.append(valid)
_gdd = gdd(temperature(row[1], 'C'), temperature(row[2], 'C'))
for year in years:
if valid.month == 2 and valid.day == 29 and year % 4 != 0:
continue
if dont_replace:
df.loc[valid.replace(year=year),
rcols[3:-1]] = (_gdd, row[4], row[5], row[6], row[7],
row[8], row[9], row[10])
continue
df.loc[valid.replace(year=year),
rcols] = (row[1], row[2], row[3], _gdd, row[4], row[5],
row[6], row[7], row[8], row[9], row[10], row[11])
if len(replaced) > 0:
print("yieldfx_workflow supplemented %s from %s to %s" %
(location, replaced[0], replaced[-1]))
# We no longer want to use rain from climodat...
return
# Go get precip from Climodat
coop = XREF[location]['climodat']
pgconn = psycopg2.connect(database='coop', host='iemdb', user='******')
cursor = pgconn.cursor()
cursor.execute(
"""
SELECT day, precip from alldata_ia where year = %s and station = %s
and day < %s
""", (jan1.year, coop, today))
for row in cursor:
valid = row[0]
pcpn = distance(row[1], 'IN').value('MM')
for year in years:
if valid.month == 2 and valid.day == 29 and year % 4 != 0:
continue
df.at[valid.replace(year=year), 'rain'] = pcpn