def data_validation_test():
pickpts = [
[-63.18, 9.8],
[-3.2, 55.5],
[47.98, 29.5],
[-97.1, 36.7],
[151.2, -33.9],
[70.25, -49.35],
[0.0, 0.0],
]
GISStemps = [27.5, 8.5, 28.0, 15.0, 18.0, 5.0, 27]
for ix in range(len(pickpts)):
lax, lox = elds.index(pickpts[ix][0], pickpts[ix][1])
wnd, wcwnd, lcwnd = get_windset(lax, lox, 2)
gloTemp = np.nanmean(teds.read(1, window=wnd))
gloSRAD = np.nanmean(hids.read(1, window=wnd)) * 1000.0 / 24.0
lc = lcds.read(1, window=lcwnd)
print("temps for ", pickpts[ix])
srad_months = []
prec_months = []
temp_months = []
for mx in range(12):
srad_months.append(
wd_filteredstats(sradds[mx].read(1, window=wcwnd), 65535))
prec_months.append(
wd_filteredstats(precds[mx].read(1, window=wcwnd), -32768))
temp_months.append(
wd_filteredstats(tavgds[mx].read(1, window=wcwnd), -3.4e38))
temp_months = np.array(temp_months)
print(GISStemps[ix], gloTemp, temp_months.mean())
print("radiations")
# pdb.set_trace()
srad_months = np.array(srad_months)
toa_months = gtu.toa_series(pickpts[ix][1])
sr12 = gtu.acc12mo_avg(srad_months)
sr_cal = sr12 / gloSRAD
gtu.arprint([gloSRAD, sr12, sr_cal])
gtu.arprint(srad_months / sr_cal)
gtu.arprint(toa_months)
prec_months = np.array(prec_months)
print("precip", prec_months.mean())
print(lc)
print("lc", lc_histo(lc))
pdb.set_trace()
rn_maxes[rnx] = rsqs[:, rnx, :].max()
if rsqs[:, rnx, :].min() < rn_mins[rnx]:
rn_mins[rnx] = rsqs[:, rnx, :].min()
if rn_trumax < rn_trus.max():
rn_trumax = rn_trus.max()
if rn_trumin > rn_trus.min():
rn_trumin = rn_trus.min()
if wc_trus_max < wc_trus.max():
wc_trus_max = wc_trus.max()
if wc_trus_min > wc_trus.min():
wc_trus_min = wc_trus.min()
print(rsqs[:, 1, :].max())
if tx + 1 == file_ct:
gtu.arprint(ins_maxes)
gtu.arprint(ins_mins)
gtu.arprint(rn_maxes)
gtu.arprint(rn_mins)
gtu.arprint([rn_trumin, rn_trumax])
gtu.arprint([wc_trus_min, wc_trus_max])
dc = {}
dc["nn_feats"] = wcb.nn_features
dc["rn_feats"] = wcb.rnn_features
dc["ins_maxes"] = ins_maxes.tolist()
dc["ins_mins"] = ins_mins.tolist()
dc["rn_maxes"] = rn_maxes
dc["rn_mins"] = rn_mins
dc["rn_true"] = [rn_trumin, rn_trumax]
dc["wc_true"] = [wc_trus_min, wc_trus_max]
rmdl.lossers,
rmdl.hypos,
rmdl.ts,
rmdl.y_trues,
rmdl.meta_ts,
rmdl.meta_losser,
]
errs, ests, step, yt, mts, met_err = sess.run(fetch, feed)
errs = np.array(errs)
# gtu.arprint([mcx, tx_, errs.mean(), errs.max(), errs.min(), met_err])
train_history.append([errs.mean(), errs.max(), errs.min(), met_err])
if tx_ == (file_ct) - 1 or tx_ % 500 == 499 or tx_ == 0:
if tx_ == 0:
gtu.arprint([mcx, tx_, errs.mean(), errs.max(), errs.min(), met_err])
else:
last500 = np.array(train_history)[-490:, :]
gtu.arprint(
[
mcx,
tx_,
"rnn:",
last500[:, 0].min(),
last500[:, 0].mean(),
last500[:, 0].max(),
]
)
gtu.arprint(
[
mcx,
def bld_eu_examp(wnd, wcwnd, lcwnd, d3lax, d3lox, lon, lat, wx,
bTrain): # an example in eng units
ex_good = True
temp_12mo = []
srad_12mo = []
prec_12mo = []
wind_12mo = []
vap_12mo = []
temp_12std = []
srad_12std = []
prec_12std = []
wind_12std = []
vap_12std = []
pwra = []
csra = []
# print(lax, lox, datct)
el = dem3ds[wx].read(1, window=wnd)
if el.shape[0] < 6 or el.shape[1] < 6:
ex_good = False
blog("bad el shape ", lat, lon)
sun_slope, ew_slope = -1.0, -1.0
else:
sun_slope, ew_slope = elev_slope(el, lat)
elev = np.nanmean(el)
elev_std = np.nanstd(el)
if elev < -100:
ex_good = False
blog("bad elev: ", lat, lon)
lc = lcds.read(1, window=lcwnd)
try:
land, water, ice, desert = lc_histo(lc)
except:
pst()
land, water, ice, desert = lc_histo(lc)
new_alb = (0.1 * water) + (0.18 * land) + (0.3 * desert) + (0.7 * ice)
if land == -1:
ex_good = False
blog("bad lwi: ", lat, lon)
barop = gtu.bp_byalt(elev)
if lat > 0:
sh1h = 0.0
nh1h = 1.0
else:
sh1h = 1.0
nh1h = 0.0
toa_12mo = np.array(gtu.toa_series(lat))
for mx in range(12):
vmean, vstd = wd_filteredstats(sradds[mx].read(1, window=wcwnd), 65535)
srad_12mo.append(vmean)
srad_12std.append(vstd)
vmean, vstd = wd_filteredstats(precds[mx].read(1, window=wcwnd),
-32768)
prec_12mo.append(vmean)
prec_12std.append(vstd)
vmean, vstd = wd_filteredstats(tavgds[mx].read(1, window=wcwnd),
-3.4e38)
temp_12mo.append(vmean)
temp_12std.append(vstd)
vmean, vstd = wd_filteredstats(windds[mx].read(1, window=wcwnd),
-3.4e38)
wind_12mo.append(vmean)
wind_12std.append(vstd)
vmean, vstd = wd_filteredstats(vapds[mx].read(1, window=wcwnd),
-3.4e38)
vap_12mo.append(vmean)
vap_12std.append(vstd)
if srad_12mo[mx] == 65535:
ex_good = False
blog("bad srad ", lat, lon)
if prec_12mo[mx] == -32768:
ex_good = False
blog("bad prec ", lat, lon)
if temp_12mo[mx] == -3.4e38:
ex_good = False
blog("bad temp ", lat, lon)
csra.append(srad_12mo[mx] * (1 - new_alb))
if csra[mx] != 0.0:
pwra.append(srad_12mo[mx] / (86.4 * csra[mx]))
else:
pwra.append(0.0)
temp_12mo = np.array(temp_12mo)
srad_12mo = np.array(srad_12mo)
prec_12mo = np.array(prec_12mo)
wind_12mo = np.array(wind_12mo)
vap_12mo = np.array(vap_12mo)
rnn_seq = [srad_12mo, prec_12mo, toa_12mo, wind_12mo, csra, pwra, vap_12mo]
toa_pwr = gtu.toaPower(lat)
wc_temp = gtu.acc12mo_avg(temp_12mo)
if wc_temp < -80 or wc_temp > 80.0:
ex_good = False
blog("bad wc_temp: ", lat, lon, wc_temp)
wc_prec = gtu.acc12mo_avg(prec_12mo)
if wc_prec < 0 or wc_prec > 3000:
ex_good = False
blog("bad wc_prec: ", lat, lon, wc_prec)
wc_srad = gtu.acc12mo_avg(srad_12mo)
if wc_srad < 0.0 or wc_srad > (600.0 * wc_radCon):
ex_good = False
blog("bad wc_srad: ", lat, lon, wc_srad)
wc_wind = gtu.acc12mo_avg(wind_12mo)
if wc_wind < 0.0 or wc_wind > 100.0:
ex_good = False
blog("bad wc_wind: ", lat, lon, wc_wind)
# pdb.set_trace()
vis_dstd = np.array(srad_12std).mean()
gtzs = (el > 0.0).sum()
ltzs = (el < 0.0).sum()
zs = (el == 0.0).sum()
pwr_ratio = wc_srad / gtu.acc12mo_avg(toa_12mo)
gtu.arprint([
lat,
lon,
wc_temp,
temp_12mo.min(),
temp_12mo.max(),
sun_slope,
ew_slope,
new_alb,
])
# print(ex_good)
ins = [
lon,
lat,
toa_pwr,
elev,
barop,
pwr_ratio,
wc_prec,
wc_srad,
wc_wind,
land,
water,
ice,
desert,
new_alb,
sh1h,
nh1h,
vis_dstd,
elev_std,
sun_slope,
ew_slope,
zs,
gtzs,
ltzs,
]
return np.array(ins), ex_good, rnn_seq, temp_12mo, wc_temp, wx
# ins[:,9].max(), ins[:,10].max(), ins[:,11].max())
# pdb.set_trace()
if errs < smerrs:
pdb.set_trace()
else:
pass # mdl.reup() # reinit and try again
fet = [smdl.psens, smdl.esens, smdl.toasens, smdl.smb, smdl.bpsens]
ps, es, toas, smbias, bpsen = mdl.sess.run(fet)
print(ps.tolist(), es.tolist(), toas.tolist(), smbias.tolist(), bpsen.tolist())
# pdb.set_trace()
for ix in range(params["batch_size"]):
# pdb.set_trace()
gtu.arprint(
[
ests[ix],
yt[ix],
smests[ix],
smhp[ix],
smep[ix],
smtp[ix],
ins[ix][2],
ins[ix][4],
ins[ix][5],
]
)
if ix % 20 == 19:
pdb.set_trace()