def load_raws_observations(obs_file,glat,glon,grid_dist_km):
"""
Loads all of the RAWS observations valid at the time in question
and converts them to Observation objects.
"""
# load observations & register them to grid
orig_obs = []
if os.path.exists(obs_file):
orig_obs = np.loadtxt(obs_file,dtype=np.object,delimiter=',')
else:
print('WARN: no observation file found.')
obss = []
omin, omax = 0.6, 0.0
# format of file
# 0 1 2 3 4 5 6 7 8 9 10 11
# yyyy,mm,dd,hh,MM,ss,lat,lon,elevation,var_id,value,variance
for oo in orig_obs:
ts = datetime(int(oo[0]),int(oo[1]),int(oo[2]),int(oo[3]),int(oo[4]),int(oo[5]),tzinfo=pytz.timezone('GMT'))
lat, lon, elev = float(oo[6]), float(oo[7]), float(oo[8])
obs, ovar = float(oo[10]), float(oo[11])
i, j = find_closest_grid_point(lat,lon,glat,glon)
# compute distance to grid points
dist_grid_pt = great_circle_distance(lon,lat,glon[i,j],glat[i,j])
# check & remove nonsense zero-variance (or negative variance) observations
if ovar > 0 and dist_grid_pt < grid_dist_km / 2.0:
obss.append(Observation(ts,lat,lon,elev,oo[9],obs,ovar,(i,j)))
omin = min(omin, obs)
omax = max(omax, obs)
print('INFO: loaded %d observations in range %g to %g [%d available]' % (len(obss),omin,omax,len(obss)))
return obss
def register_to_grid(self, wrf_data):
"""
Find the nearest grid point to the current location.
"""
# only co-register to grid if required
mlon, mlat = wrf_data.get_lons(), wrf_data.get_lats()
self.grid_pt = find_closest_grid_point(self.lon, self.lat, mlon, mlat)
self.dist_grid_pt = great_circle_distance(self.lon, self.lat,
mlon[self.grid_pt], mlat[self.grid_pt])
def register_to_grid(self, wrf_data):
"""
Find the nearest grid point to the current location.
"""
# only co-register to grid if required
mlon, mlat = wrf_data.get_lons(), wrf_data.get_lats()
self.grid_pt = find_closest_grid_point(self.lon, self.lat, mlon, mlat)
self.dist_grid_pt = great_circle_distance(self.lon, self.lat,
mlon[self.grid_pt], mlat[self.grid_pt])
def simple_kriging_data_to_model(obs_data, obs_stds, mu_mod, wrf_data,
mod_stds, t):
"""
Simple kriging of data points to model points. The kriging results in
the matrix K, which contains the kriged observations and the matrix V,
which contains the kriging variance.
"""
mlons, mlats = wrf_data.get_lons(), wrf_data.get_lats()
K = np.zeros_like(mlons)
V = np.zeros_like(mlons)
Nobs = len(obs_data)
obs_vals = np.zeros((Nobs, ))
station_lonlat = []
gridndx = []
mu_obs = np.zeros((Nobs, ))
measV = np.zeros((Nobs, ))
# accumulate the indices of the nearest grid points
ndx = 0
for obs in obs_data:
gridndx.append(obs.get_nearest_grid_point())
obs_vals[ndx] = obs.get_value()
station_lonlat.append(obs.get_position())
mu_obs[ndx] = mu_mod[obs.get_nearest_grid_point()]
measV[ndx] = obs.get_measurement_variance()
ndx += 1
# compute observation residuals (using model fit from examine_station_data)
res_obs = obs_vals - mu_obs
# construct the covariance matrix and invert it
C = np.asmatrix(construct_spatial_correlation_matrix2(station_lonlat))
oS = np.asmatrix(np.diag(obs_stds))
Sigma = oS.T * C * oS + np.diag(measV)
SigInv = np.linalg.inv(Sigma)
diagnostics().push("skdm_cov_cond", np.linalg.cond(Sigma))
# run the kriging estimator for each model grid point
K = np.zeros_like(mlats)
cov = np.zeros_like(mu_obs)
for p in np.ndindex(K.shape):
# compute the covariance array anew for each grid point
for k in range(Nobs):
lon, lat = station_lonlat[k]
cc = max(
0.8565 -
0.0063 * great_circle_distance(mlons[p], mlats[p], lon, lat),
0.0)
cov[k] = mod_stds[p] * cc * obs_stds[k]
csi = np.dot(cov, SigInv)
K[p] = mu_mod[p] + np.dot(csi, res_obs)
V[p] = mod_stds[p]**2 - np.dot(csi, cov)
return K, V
def get_grid_values_from_file(ncpath,varname,lat,lon): d = netCDF4.Dataset(ncpath) glat, glon = d.variables['Lat'][:,:], d.variables['Lon'][:,:] V = d.variables[varname] i, j = find_closest_grid_point(lon, lat, glon, glat) dist = great_circle_distance(glon[i,j],glat[i,j],lon,lat) vals = V[i,j,...] d.close() return dist, i, j, vals
def get_ngp(lat,lon,ncpath):
"""
Finds the closest grid point to lat/lon, prints the indices and shows the distance to the point.
"""
d = netCDF4.Dataset(ncpath)
# find closest grid point
glat,glon = d.variables['XLAT'][0,:,:], d.variables['XLONG'][0,:,:]
i,j = find_closest_grid_point(lon,lat,glon,glat)
dist = great_circle_distance(lon,lat,glon[i,j],glat[i,j])
print('GP closest to lat/lon %g,%g is %d,%d with distance %g km.' % (lat,lon,i,j,dist))
def simple_kriging_data_to_model(obs_data, obs_stds, mu_mod, wrf_data, mod_stds, t):
"""
Simple kriging of data points to model points. The kriging results in
the matrix K, which contains the kriged observations and the matrix V,
which contains the kriging variance.
"""
mlons, mlats = wrf_data.get_lons(), wrf_data.get_lats()
K = np.zeros_like(mlons)
V = np.zeros_like(mlons)
Nobs = len(obs_data)
obs_vals = np.zeros((Nobs,))
station_lonlat = []
gridndx = []
mu_obs = np.zeros((Nobs,))
measV = np.zeros((Nobs,))
# accumulate the indices of the nearest grid points
ndx = 0
for obs in obs_data:
gridndx.append(obs.get_nearest_grid_point())
obs_vals[ndx] = obs.get_value()
station_lonlat.append(obs.get_position())
mu_obs[ndx] = mu_mod[obs.get_nearest_grid_point()]
measV[ndx] = obs.get_measurement_variance()
ndx += 1
# compute observation residuals (using model fit from examine_station_data)
res_obs = obs_vals - mu_obs
# construct the covariance matrix and invert it
C = np.asmatrix(construct_spatial_correlation_matrix2(station_lonlat))
oS = np.asmatrix(np.diag(obs_stds))
Sigma = oS.T * C * oS + np.diag(measV)
SigInv = np.linalg.inv(Sigma)
diagnostics().push("skdm_cov_cond", np.linalg.cond(Sigma))
# run the kriging estimator for each model grid point
K = np.zeros_like(mlats)
cov = np.zeros_like(mu_obs)
for p in np.ndindex(K.shape):
# compute the covariance array anew for each grid point
for k in range(Nobs):
lon, lat = station_lonlat[k]
cc = max(0.8565 - 0.0063 * great_circle_distance(mlons[p], mlats[p], lon, lat), 0.0)
cov[k] = mod_stds[p] * cc * obs_stds[k]
csi = np.dot(cov, SigInv)
K[p] = mu_mod[p] + np.dot(csi, res_obs)
V[p] = mod_stds[p] ** 2 - np.dot(csi, cov)
return K, V
def get_ngp(lat, lon, ncpath):
"""
Finds the closest grid point to lat/lon, prints the indices and shows the distance to the point.
"""
d = netCDF4.Dataset(ncpath)
# find closest grid point
glat, glon = d.variables['XLAT'][0, :, :], d.variables['XLONG'][0, :, :]
i, j = find_closest_grid_point(lon, lat, glon, glat)
dist = great_circle_distance(lon, lat, glon[i, j], glat[i, j])
print('GP closest to lat/lon %g,%g is %d,%d with distance %g km.' %
(lat, lon, i, j, dist))
def load_raws_observations(obs_file, glat, glon, grid_dist_km):
"""
Loads all of the RAWS observations valid at the time in question
and converts them to Observation objects.
"""
# load observations & register them to grid
orig_obs = []
if os.path.exists(obs_file):
orig_obs = np.loadtxt(obs_file, dtype=np.object, delimiter=',')
else:
print('WARN: no observation file found.')
obss = []
omin, omax = 0.6, 0.0
# format of file
# 0 1 2 3 4 5 6 7 8 9 10 11
# yyyy,mm,dd,hh,MM,ss,lat,lon,elevation,var_id,value,variance
for oo in orig_obs:
ts = datetime(int(oo[0]),
int(oo[1]),
int(oo[2]),
int(oo[3]),
int(oo[4]),
int(oo[5]),
tzinfo=pytz.timezone('GMT'))
lat, lon, elev = float(oo[6]), float(oo[7]), float(oo[8])
obs, ovar = float(oo[10]), float(oo[11])
i, j = find_closest_grid_point(lat, lon, glat, glon)
# compute distance to grid points
dist_grid_pt = great_circle_distance(lon, lat, glon[i, j], glat[i, j])
# check & remove nonsense zero-variance (or negative variance) observations
if ovar > 0 and dist_grid_pt < grid_dist_km / 2.0:
obss.append(
Observation(ts, lat, lon, elev, oo[9], obs, ovar, (i, j)))
omin = min(omin, obs)
omax = max(omax, obs)
print('INFO: loaded %d observations in range %g to %g [%d available]' %
(len(obss), omin, omax, len(obss)))
return obss
if len(obs_at_t) > 0:
# construct pairwise dataset
dists = []
sqdiffs = []
for i in range(len(obs_at_t)):
sid = obs_at_t[i].get_station().get_id()
pi = obs_at_t[i].get_position()
stats = station_stats[sid]
oi = obs_at_t[i].get_value()
if cfg['standardize']:
oi = (oi - stats[0]) / stats[1]
for j in range(i+1, len(obs_at_t)):
pj = obs_at_t[j].get_position()
oj = obs_at_t[j].get_value()
dist = great_circle_distance(pi[0], pi[1], pj[0], pj[1])
if dist < max_dist:
dists.append(dist)
sqdiffs.append(0.5 * (oi - oj)**2)
all_dists.append(dist)
all_sqdiffs.append(0.5 * (oi - oj)**2)
fname = 'std_variogram_est_%03d.png' % t if cfg['standardize'] else 'variogram_est_%03d.png' % t
plot_variogram(dists, sqdiffs, bins, 'Variogram at time %s' % str(t_now),
os.path.join(cfg['output_dir'], fname))
fname = 'std_variogram_est_all.png' if cfg['standardize'] else 'variogram_est_all.png'
plot_variogram(all_dists, all_sqdiffs, bins, 'Variogram (all observations)',
os.path.join(cfg['output_dir'], fname))
# hack to plot the lower part of the variogram
def run_data_assimilation(in_dir0, in_dir1, fm_dir):
# load RTMA data for previous
print("INFO: loading RTMA data for time t-1 from [%s] ..." % in_dir0)
tm0 = time_from_dir(in_dir0)
tm = time_from_dir(in_dir1)
max_back = 6
data0 = None
while max_back > 0:
in_dir0 = 'inputs/%04d%02d%02d-%02d00' % (tm0.year, tm0.month, tm0.day, tm0.hour)
print('INFO: searching for RTMA in directory %s' % in_dir0)
data0 = load_rtma_data(in_dir0)
if data0 is not None:
break
print("WARN: cannot find RTMA data for time %s in directory %s, going back one hour" % (str(tm0),in_dir0))
max_back -= 1
tm0 = tm0 - timedelta(0,3600)
if data0 is None:
print("FATAL: cannot find a suitable previous RTMA analysis fror time %s." % str(tm))
return
print("INFO: loading RTMA data for time t from [%s] ..." % in_dir1)
data1 = load_rtma_data(in_dir1)
if data1 is None:
print('FATAL: insufficient environmnetal data for time %s, skipping ...' % tm)
return
# retrieve variables from RTMA
lat, lon, hgt = data0['Lat'], data0['Lon'], data0['HGT']
t20, relh0 = data0['T2'], data0['RH']
t21, relh1, rain = data1['T2'], data1['RH'], data1['RAIN']
ed0, ew0 = compute_equilibria(t20,relh0)
ed1, ew1 = compute_equilibria(t21,relh1)
tm0, tm = data0['Time'], data1['Time']
tm_str = tm.strftime('%Y%m%d-%H00')
tm_str0 = tm0.strftime('%Y%m%d-%H00')
# compute mean values for the Equilibria at t-1 and at t
ed = 0.5 * (ed0 + ed1)
ew = 0.5 * (ew0 + ew1)
dom_shape = lat.shape
print('INFO: domain size is %d x %d grid points.' % dom_shape)
print('INFO: domain extent is lats (%g to %g) lons (%g to %g).' % (np.amin(lat),np.amax(lat),np.amin(lon),np.amax(lon)))
print('INFO: stepping from time %s to time %s' % (tm0, tm))
# initialize output file
out_fm_file = os.path.join(fm_dir, 'fm-%s.nc' % tm_str)
out_file = netCDF4.Dataset(out_fm_file, 'w')
out_file.createDimension('fuel_moisture_classes_stag', 5)
out_file.createDimension('south_north', dom_shape[0])
out_file.createDimension('west_east', dom_shape[1])
nced = out_file.createVariable('Ed', 'f4', ('south_north', 'west_east'))
nced[:,:] = ed
ncew = out_file.createVariable('Ew', 'f4', ('south_north', 'west_east'))
ncew[:,:] = ew
ncfmc_fc = out_file.createVariable('FMC_GC_FC', 'f4', ('south_north', 'west_east','fuel_moisture_classes_stag'))
ncfmc_an = out_file.createVariable('FMC_GC', 'f4', ('south_north', 'west_east','fuel_moisture_classes_stag'))
nckg = out_file.createVariable('K', 'f4', ('south_north', 'west_east','fuel_moisture_classes_stag'))
ncfmc_cov = out_file.createVariable('FMC_COV', 'f4', ('south_north', 'west_east','fuel_moisture_classes_stag', 'fuel_moisture_classes_stag'))
ncrelh = out_file.createVariable('RELH','f4', ('south_north', 'west_east'))
ncrelh[:,:] = relh1
nctemp = out_file.createVariable('T2','f4', ('south_north', 'west_east'))
nctemp[:,:] = t21
nclat = out_file.createVariable('Lat', 'f4', ('south_north', 'west_east'))
nclat[:,:] = lat
nclon = out_file.createVariable('Lon', 'f4', ('south_north', 'west_east'))
nclon[:,:] = lon
print('INFO: opened %s and wrote XLAT,XLONG,RELH,T2,Ed,Ew fields.' % out_fm_file)
### Load observation data from the stations
# compute the diagonal distance between grid points
grid_dist_km = great_circle_distance(lon[0,0], lat[0,0], lon[1,1], lat[1,1])
print('INFO: diagonal distance in grid is %g' % grid_dist_km)
raws_path = os.path.join(in_dir1, 'raws_ingest_%4d%02d%02d-%02d%02d.csv' % (tm.year,tm.month,tm.day,tm.hour,tm.minute))
obss = load_raws_observations(raws_path,lat,lon)
print('INFO: Loaded %d observations.' % (len(obss)))
# set up parameters
Nk = 3 # we simulate 4 types of fuel
Q = np.diag([1e-4,5e-5,1e-5,1e-6,1e-6])
P0 = np.diag([0.01,0.01,0.01,0.001,0.001])
Tk = np.array([1.0, 10.0, 100.0])
dt = (tm - tm0).seconds
print("INFO: Time step is %d seconds." % dt)
# remove rain that is too small to make any difference
rain[rain < 0.01] = 0
# preprocess all covariates
X = np.zeros((dom_shape[0], dom_shape[1], 4))
X[:,:,1] = 1.0
X[:,:,2] = hgt / 2000.0
if np.any(rain) > 0.01:
X[:,:,3] = rain
else:
X = X[:,:,:3]
# load current state (or initialize from equilibrium if not found)
fm0 = None
fm_cov0 = None
in_fm_file = os.path.join(fm_dir, 'fm-%s.nc' % tm_str0)
if os.path.isfile(in_fm_file):
in_file = netCDF4.Dataset(in_fm_file, 'r')
fm0 = in_file.variables['FMC_GC'][:,:,:]
fm_cov0 = in_file.variables['FMC_COV'][:,:,:,:]
print('INFO: found input file %s, initializing from it [fm is %dx%dx%d, fm_cov is %dx%dx%dx%d]' %
(in_fm_file,fm0.shape[0],fm0.shape[1],fm0.shape[2],fm_cov0.shape[0],fm_cov0.shape[1],
fm_cov0.shape[2],fm_cov0.shape[3]))
in_file.close()
else:
print('INFO: input file %s not found, initializing from equilibrium' % in_fm_file)
fm0 = 0.5 * (ed + ew)
fm0 = fm0[:,:,np.newaxis][:,:,np.zeros((5,),dtype=np.int)]
fm0[:,:,3] = -0.04
fm0[:,:,4] = 0
fm_cov0 = P0
models = GridMoistureModel(fm0, Tk, 0.08, 2, 0.6, 7, fm_cov0)
print('INFO: performing forecast at: [time=%s].' % str(tm))
# compute the FORECAST
models.advance_model(ed, ew, rain, dt, Q)
f = models.get_state()
ncfmc_fc[:,:,:] = f
# fill 10-hr forecast as the first field of X
X[:,:,0] = f[:,:,1]
# examine the assimilated fields (if assimilation is activated)
for i in range(3):
print('INFO [%d]: [min %g, mean %g, max %g]' % (i, np.amin(f[:,:,i]), np.mean(f[:,:,i]), np.amax(f[:,:,i])))
if np.any(f[:,:,i] < 0.0):
print("WARN: in field %d there were %d negative moisture values !" % (i, np.count_nonzero(f[:,:,i] < 0.0)))
if np.any(f[:,:,i] > 0.6):
print("WARN: in field %d there were %d moisture values above 0.6!" % (i, np.count_nonzero(f[:,:,i] > 2.5)))
if len(obss) > 0:
print('INFO: running trend surface model ...')
# fit the trend surface model to data
tsm, tsm_var, s2 = fit_tsm(obss, X)
print('INFO: microscale variability variance is %g' % s2)
if np.count_nonzero(tsm > 0.6) > 0:
print('WARN: in TSM found %d values over 0.6, %d of those had rain, clamped to 2.5' %
(np.count_nonzero(tsm > 0.6),
np.count_nonzero(np.logical_and(tsm > 0.6, rain > 0.0))))
tsm[tsm > 0.6] = 0.6
if np.count_nonzero(tsm < 0.0) > 0:
print('WARN: in TSM found %d values under 0.0, clamped to 0.0' % np.count_nonzero(tsm < 0.0))
tsm[tsm < 0.0] = 0.0
print('INFO: running KF ...')
# run the kalman update step
Kg = np.zeros((dom_shape[0], dom_shape[1], len(Tk)+2))
models.kalman_update_single2(tsm[:,:,np.newaxis], tsm_var[:,:,np.newaxis,np.newaxis], 1, Kg)
# check post-assimilation results
f = models.get_state()
for i in range(3):
if np.any(f[:,:,i] < 0.0):
print("WARN: in field %d there were %d negative moisture values and %f values over 0.6 !" %
(i, np.count_nonzero(f[:,:,i] < 0.0),np.count_nonzero(f[:,:,i] > 0.6)))
f[f < 0] = 0
nckg[:,:,:] = Kg
print('INFO: storing results in netCDF file %s.' % out_fm_file)
# store post-assimilation (or forecast depending on whether observations were available) FM-10 state and variance
ncfmc_an[:,:,:] = f
ncfmc_cov[:,:,:,:] = models.get_state_covar()
# close the netCDF file (relevant if we did write into FMC_GC)
out_file.close()
print('INFO: SUCCESS')
# compute COVARIANCE between station residuals and plot this vs. distance
Ns = len(stations)
ss = [s.get_name() for s in stations]
C = np.zeros((Ns,Ns))
D = np.zeros((Ns,Ns))
WD = np.zeros((Ns,Ns))
E = np.zeros((Ns,Ns))
for i in range(Ns):
r1, (lon1, lat1) = residuals[ss[i]], stations[i].get_position()
i1,j1 = stations[i].get_nearest_grid_point()
for j in range(Ns):
r2, (lon2, lat2) = residuals[ss[j]], stations[j].get_position()
i2,j2 = stations[j].get_nearest_grid_point()
cc = np.cov(r1, r2)
C[i,j] = cc[0, 1]
D[i,j] = great_circle_distance(lon1, lat2, lon2, lat2)
WD[i,j] = great_circle_distance(lon[i1,j1], lat[i1,j1], lon[i2,j2], lat[i2,j2])
E[i,j] = np.abs(stations[i].get_elevation() - stations[j].get_elevation()) / 1000.0
f = plt.figure(figsize = (16,16))
f.subplots_adjust(hspace = 0.5, wspace = 0.5)
ax = plt.subplot(221)
plt.imshow(C, interpolation = 'nearest')
plt.title('Covariance [-]')
plt.colorbar()
ax.set_xticks(np.arange(len(ss)))
ax.set_xticklabels(ss, rotation = 90)
ax.set_yticks(np.arange(len(ss)))
ax.set_yticklabels(ss)
ax = plt.subplot(222)
plt.imshow(D, interpolation = 'nearest')
map(string.strip, si_list))
stations = {}
for code in si_list:
mws = MesoWestStation(code)
mws.load_station_info(
os.path.join(cfg["station_info_dir"], "%s.info" % code))
stations[code] = mws
# first construct a distance matrix for all stations
st_dists = np.zeros((len(sids_list), len(sids_list)))
for j in range(len(sids_list)):
lonj, latj = stations[sids_list[j]].get_position()
for k in range(j + 1, len(sids_list)):
lonk, latk = stations[sids_list[k]].get_position()
d = great_circle_distance(lonj, latj, lonk, latk)
st_dists[j, k] = d
st_dists[k, j] = d
# accumulate data over all time points
for i in range(N):
dists_i = []
sqdiffs_i = []
# retrieve valid measurements, rest is nan
di = data[i]
dobs = di['kriging_obs']
Nobs = len(dobs)
obs = np.zeros(len(sids_list))
def run_module():
# read in configuration file to execute run
print("Reading configuration from [%s]" % sys.argv[1])
with open(sys.argv[1]) as f:
cfg = eval(f.read())
# init diagnostics
init_diagnostics(
os.path.join(cfg['output_dir'], 'moisture_model_v1_diagnostics.txt'))
diagnostics().configure_tag("s2_eta_hat", True, True, True)
diagnostics().configure_tag("kriging_rmse", True, True, True)
diagnostics().configure_tag("kriging_beta", True, True, True)
diagnostics().configure_tag("kriging_iters", False, True, True)
diagnostics().configure_tag("kriging_subzero_s2_estimates", False, True,
True)
# load the wrfinput file
wrfin = WRFModelData(cfg['wrf_input'],
['T2', 'Q2', 'PSFC', 'HGT', 'FMC_GC', 'FMEP'])
lat, lon = wrfin.get_lats(), wrfin.get_lons()
ts_now = wrfin['GMT'][0]
dom_shape = lat.shape
print('INFO: domain size is %d x %d grid points, wrfinput timestamp %s' %
(dom_shape[0], dom_shape[1], str(ts_now)))
print('INFO: domain extent is lats (%g to %g) lons (%g to %g).' %
(np.amin(lat), np.amax(lat), np.amin(lon), np.amax(lon)))
# compute the diagonal distance between grid points
grid_dist_km = great_circle_distance(lon[0, 0], lat[0, 0], lon[1, 1],
lat[1, 1])
print('INFO: diagonal distance in grid is %g' % grid_dist_km)
# load observations but discard those too far away from the grid nodes
obss = load_raws_observations(cfg['observations'], lat, lon, grid_dist_km)
fm10 = build_observation_data(obss)
print('INFO: %d different time instances found in observations' %
len(fm10))
# if a previous cycle is available (i.e. the wrfoutput is a valid file)
if os.path.exists(cfg['wrf_output_prev']) and check_overlap(
cfg['wrf_output_prev'], ts_now):
# load the model as a wrfout with all default variables
wrfout = WRFModelData(cfg['wrf_output_prev'])
outts = wrfout['GMT']
print("INFO: previous forecast [%s - %s] exists, running DA till %s" %
(str(outts[0]), str(outts[-1]), str(ts_now)))
# run from the start until now (retrieve fuel moisture, extended parameters, covariance matrix)
model = run_data_assimilation(wrfout, fm10, ts_now, cfg)
# store this for the current time instance (fm, ep in the wrfinput, P next to it)
d = netCDF4.Dataset(cfg['wrf_input'], 'r+')
d.variables['FMC_GC'] = fm
d.variables['FMEP'] = ep
d.close()
# store the covariance matrix alongside the wrfinput file
dir = os.path.dirname(wrfin)
store_covariance_matrix(P, os.path.join(dir, 'P.nc'))
else:
print(
"INFO: no previous forecast found, running DA from equilibrium at %s"
% (str(ts_now)))
# initialize from weather equilibrium and perform one DA step
model = init_from_equilibrium(wrfin, fm10, ts_now, cfg)
# store result in wrfinput dataset
d = netCDF4.Dataset(cfg['wrf_input'], 'r+')
fmcep = model.get_state()
d.variables['FMC_GC'][0, :3, :, :] = fmcep[:, :, :3].transpose(
(2, 0, 1))
d.variables['FMEP'][0, :, :, :] = fmcep[:, :, 3:5].transpose((2, 0, 1))
d.close()
store_covariance_matrix(
model.get_state_covar(),
os.path.join(os.path.dirname(cfg['wrf_input']), 'P.nc'))
return 0
def run_module():
# read in configuration file to execute run
print("Reading configuration from [%s]" % sys.argv[1])
with open(sys.argv[1]) as f:
cfg = eval(f.read())
# ensure output path exists
if not os.path.isdir(cfg['output_dir']):
os.mkdir(cfg['output_dir'])
# configure diagnostics
init_diagnostics(os.path.join(cfg['output_dir'], 'moisture_model_v1_diagnostics.txt'))
# Trend surface model diagnostics
diagnostics().configure_tag("kriging_cov_cond", True, True, True)
diagnostics().configure_tag("s2_eta_hat", True, True, True)
diagnostics().configure_tag("kriging_rmse", True, True, True)
diagnostics().configure_tag("kriging_beta", True, True, True)
diagnostics().configure_tag("kriging_iters", False, True, True)
diagnostics().configure_tag("kriging_subzero_s2_estimates", False, True, True)
diagnostics().configure_tag("fm10_kriging_var", True, True, True)
diagnostics().configure_tag("f0_summary", True, True, True)
diagnostics().configure_tag("f1_summary", True, True, True)
diagnostics().configure_tag("f2_summary", True, True, True)
diagnostics().configure_tag("f3_summary", True, True, True)
# Assimilation parameters
diagnostics().configure_tag("K0_summary", True, True, True)
diagnostics().configure_tag("K1_summary", True, True, True)
diagnostics().configure_tag("K2_summary", True, True, True)
diagnostics().configure_tag("K3_summary", True, True, True)
diagnostics().configure_tag("assim_info", False, False, True)
# Model forecast, analysis and non-assimilated model: state, covariance, errors
diagnostics().configure_tag("fm10f_rmse", True, True, True)
diagnostics().configure_tag("fm10na_rmse", True, True, True)
# all simulation times and all assimilation times (subset)
diagnostics().configure_tag("mta", False, True, True)
diagnostics().configure_tag("mt", False, True, True)
# observation values and their nearest grid points
diagnostics().configure_tag("obs_vals", False, True, True)
diagnostics().configure_tag("obs_ngp", False, True, True)
# in test mode, we will emit observations at the target station
# our predictions, the nearest grid point and the test station id
diagnostics().configure_tag("test_obs", True, True, True)
diagnostics().configure_tag("test_pred", True, True, True)
diagnostics().configure_tag("test_ngp", True, True, True)
diagnostics().configure_tag("test_station_id", True, True, True)
### Load and preprocess WRF model data
# load WRF data
wrf_data = WRFModelData(cfg['wrf_output'], ['T2', 'Q2', 'PSFC', 'RAINNC', 'RAINC', 'HGT'])
wrf_data.slice_field('HGT')
# read in spatial and temporal extent of WRF variables
lat, lon = wrf_data.get_lats(), wrf_data.get_lons()
hgt = wrf_data['HGT']
tm = wrf_data.get_gmt_times()
Nt = cfg['Nt'] if cfg.has_key('Nt') and cfg['Nt'] is not None else len(tm)
dom_shape = lat.shape
print('INFO: domain size is %d x %d grid points.' % dom_shape)
print('INFO: domain extent is lats (%g to %g) lons (%g to %g).' % (np.amin(lat),np.amax(lat),np.amin(lon),np.amax(lon)))
# if writing is requested, open output file and set up dimensions
if cfg['write_fields'] not in [ 'all', 'fmc_gc', 'none']:
error('FATAL: write_fields must be one of all, fmc_gc or none.')
if cfg['write_fields'] == 'none':
cfg['write_fields'] = False
out_file = None
ncfmc_gc, ncfm10a, ncfm10aV, ncfm10f, cnfm10fV, ncfm10na = None, None, None, None, None, None
nctsmV, ncKg = None, None
if cfg['write_fields']:
out_file = netCDF4.Dataset(cfg['output_dir'] + '/fields.nc', 'w')
out_file.createDimension('Time', None)
out_file.createDimension('fuel_moisture_classes_stag', 5)
out_file.createDimension('south_north', dom_shape[0])
out_file.createDimension('west_east', dom_shape[1])
ncfmc_gc = out_file.createVariable('FMC_GC', 'f4', ('Time', 'fuel_moisture_classes_stag', 'south_north', 'west_east'))
if cfg['write_fields'] == 'all':
ncfm10a = out_file.createVariable('fm10a', 'f4', ('Time', 'south_north', 'west_east'))
ncfm10aV = out_file.createVariable('fm10a_var', 'f4', ('Time', 'south_north', 'west_east'))
ncfm10na = out_file.createVariable('fm10na', 'f4', ('Time', 'south_north', 'west_east'))
ncfm10f = out_file.createVariable('fm10f', 'f4', ('Time', 'south_north', 'west_east'))
ncfm10fV = out_file.createVariable('fm10f_var', 'f4', ('Time', 'south_north', 'west_east'))
nctsmV = out_file.createVariable('tsm_var', 'f4', ('Time', 'south_north', 'west_east'))
ncKg = out_file.createVariable('kalman_gain', 'f4', ('Time', 'south_north', 'west_east'))
print('INFO: opened fields.nc for writing ALL output fields.')
else:
print('INFO: opened field.nc for writing FMC_GC only.')
test_mode = (cfg['run_mode'] == 'test')
tgt_station = None
if cfg['run_mode'] == 'test':
print('INFO: running in TEST mode! Will perform leave-one-out tesing.')
tgt_station_id = cfg['target_station_id']
diagnostics().push('test_station_id', tgt_station_id)
elif cfg['run_mode'] == 'production':
print('INFO: running in PRODUCTION mode! Using all observation stations.')
else:
error('FATAL: invalid run mode! Must be "test" or "production".')
# determine simulation times
tm_start = parse_datetime(cfg['start_time']) if cfg['start_time'] is not None else tm[0]
tm_end = parse_datetime(cfg['end_time']) if cfg['end_time'] is not None else tm[-1]
# if the required start time or end time are outside the simulation domain, exit with an error
if tm_start < tm[0] or tm_end > tm[-1]:
print('FATAL: invalid time range, required [%s-%s], availble [%s-%s]' %
(str(tm_start), str(tm_end), str(tm[0]), str(tm[-1])))
sys.exit(2)
print('INFO: time limits are %s to %s\nINFO: simulation is from %s to %s' %
(str(tm_start), str(tm_end), str(tm[0]), str(tm[-1])))
# retrieve dynamic covariates and remove mean at each time point for T2 and PSFC
T2 = wrf_data['T2']
#T2 -= np.mean(np.mean(T2,axis=0),axis=0)[np.newaxis,np.newaxis,:]
PSFC = wrf_data['PSFC']
#PSFC -= np.mean(np.mean(PSFC,axis=0),axis=0)[np.newaxis,np.newaxis,:]
# numerical fix - if it rains at an intensity of less than 0.001 per hour, set rain to zero
# also, use log(rain + 1) to prevent wild trend surface model predictions when stations see little rain
# but elsewhere there is too much rain
# without this, numerical errors in trend surface model may pop up
rain = wrf_data['RAIN']
#rain[rain < 0.01] = 0.0
rain = np.log(rain + 1.0)
# moisture equilibria are now computed from averaged Q,P,T at beginning and end of period
Ed, Ew = wrf_data.get_moisture_equilibria()
### Load observation data from the stations
# compute the diagonal distance between grid points
grid_dist_km = great_circle_distance(lon[0,0], lat[0,0], lon[1,1], lat[1,1])
print('INFO: diagonal distance in grid is %g' % grid_dist_km)
# load station data from files
with open(cfg['station_list_file'], 'r') as f:
si_list = f.read().split('\n')
si_list = filter(lambda x: len(x) > 0 and x[0] != '#', map(string.strip, si_list))
# for each station id, load the station
stations = []
for code in si_list:
mws = MesoWestStation(code)
mws.load_station_info(os.path.join(cfg["station_info_dir"], "%s.info" % code))
mws.register_to_grid(wrf_data)
if mws.get_dist_to_grid() < grid_dist_km / 2.0:
print('Station %s: lat %g lon %g nearest grid pt %s lat %g lon %g dist_to_grid %g' %
(code, mws.lat, mws.lon, str(mws.grid_pt), lat[mws.grid_pt], lon[mws.grid_pt], mws.dist_grid_pt))
mws.load_station_data(os.path.join(cfg["station_data_dir"], "%s.obs" % code))
if test_mode and mws.get_id() == tgt_station_id:
tgt_station = mws
print('INFO: in test mode, targeting station %s (removed from data pool).' % tgt_station_id)
diagnostics().push("test_ngp", mws.get_nearest_grid_point())
else:
stations.append(mws)
print('Loaded %d stations (discarded %d stations, too far from grid).' % (len(stations), len(si_list) - len(stations)))
if test_mode and tgt_station is None:
error('FATAL: in test mode, a station was removed that was not among accepted stations.')
# build the observation data
obs_data_fm10 = build_observation_data(stations, 'FM')
# build target data if in test mode
tgt_obs_fm10 = None
test_ngp = None
if test_mode:
test_ngp = tgt_station.get_nearest_grid_point()
tgt_obs_fm10 = build_observation_data([tgt_station], 'FM')
### Initialize model and visualization
# construct initial conditions from timestep 0
E = 0.5 * (Ed[0,:,:] + Ew[0,:,:])
# set up parameters
Nk = 4 # we simulate 4 types of fuel
Q = np.diag(cfg['Q'])
P0 = np.diag(cfg['P0'])
Tk = np.array([1.0, 10.0, 100.0, 1000.0]) * 3600
dt = (tm[1] - tm[0]).seconds
print("INFO: Computed timestep from WRF is is %g seconds." % dt)
mresV = np.zeros_like(E)
mid = np.zeros_like(E)
Kg = np.zeros((dom_shape[0], dom_shape[1], len(Tk)+2))
# preprocess all static covariates
cov_ids = cfg['covariates']
Xd3 = len(cov_ids) + 1
X = np.zeros((dom_shape[0], dom_shape[1], Xd3))
Xr = np.zeros((dom_shape[0], dom_shape[1], Xd3))
static_covar_map = { "lon" : lon - np.mean(lon), "lat" : lat - np.mean(lat), "elevation" : hgt - np.mean(hgt), "constant" : np.ones(dom_shape) }
dynamic_covar_map = { "temperature" : T2, "pressure" : PSFC, "rain" : rain }
for i in range(1, Xd3):
cov_id = cov_ids[i-1]
if cov_id in static_covar_map:
print('INFO: found static covariate %s' % cov_id)
Xr[:, :, i] = static_covar_map[cov_id]
elif cov_id in dynamic_covar_map:
print('INFO: found dynamic covariate %s' % cov_id)
else:
print('FATAL: unknown covariate %s encountered' % cov_id)
sys.exit(2)
print("INFO: there are %d covariates (including model state)" % Xd3)
# retrieve assimilation time window
assim_time_win = cfg['assimilation_time_window']
print('GMM init: equilibrium (%g,%g,%g) and at 86,205 %g' % (np.amin(E),np.mean(E),np.amax(E),E[86,205]))
models = GridMoistureModel(E[:,:,np.newaxis][:,:,np.zeros((4,),dtype=np.int)], Tk, P0)
models_na = GridMoistureModel(E[:,:,np.newaxis][:,:,np.zeros((4,),dtype=np.int)], Tk, P0)
### Run model for each WRF timestep and assimilate data when available
t_start, t_end = 1, len(tm)-1
while tm_start > tm[t_start]:
t_start+=1
while tm_end < tm[t_end]:
t_end-=1
# the first FMC_GC value gets filled out with equilibria
if cfg['write_fields']:
for i in range(Nk):
ncfmc_gc[0, i, :, :] = E
print('INFO: running simulation from %s (%d) to %s (%d).' % (str(tm[t_start]), t_start, str(tm[t_end]), t_end))
for t in range(t_start, t_end+1):
model_time = tm[t]
print("INFO: time: %s, step: %d" % (str(model_time), t))
diagnostics().push("mt", model_time)
models_na.advance_model(Ed[t-1,:,:], Ew[t-1,:,:], rain[t-1,:,:], dt, Q)
models.advance_model(Ed[t-1,:,:], Ew[t-1,:,:], rain[t-1,:,:], dt, Q)
# extract fuel moisture contents [make a fresh copy every iteration!]
f = models.get_state().copy()
f_na = models_na.get_state().copy()
# push 10-hr fuel state & variance of forecast
if cfg['write_fields'] == 'all':
ncfm10f[t,:,:] = models.get_state()[:,:,1]
ncfm10fV[t,:,:] = models.P[:,:,1,1]
ncfm10na[t,:,:] = models_na.get_state()[:,:,1]
# examine the assimilated fields (if assimilation is activated)
for i in range(4):
diagnostics().push("f%d_summary" % i, (t, np.amin(f[:,:,i]), np.mean(f[:,:,i]), np.amax(f[:,:,i])))
if np.any(f[:,:,i] < 0.0):
print("WARN: in field %d there were %d negative moisture values !" % (i, np.count_nonzero(f[:,:,i] < 0.0)))
ind = np.unravel_index(np.argmin(f[:,:,i]), f.shape[:2])
print(models.P[ind[0],ind[1],:,:])
print("Full model state at position %d,%d:" % (ind[0],ind[1]))
print(models.m_ext[ind[0],ind[1],:])
if np.any(f[:,:,i] > 2.5):
print("WARN: in field %d there were %d moisture values above 2.5!" % (i, np.count_nonzero(f[:,:,i] > 2.5)))
ind = np.unravel_index(np.argmax(f[:,:,i]), f.shape[:2])
print(models.P[ind[0],ind[1],:,:])
print("Full model state at position %d,%d:" % (ind[0],ind[1]))
print(models.m_ext[ind[0],ind[1],:])
if cfg['assimilate']:
# run Kriging on each observed fuel type
Kfs, Vfs, fns = [], [], []
for obs_data, fuel_ndx in [ (obs_data_fm10, 1) ]:
# run the kriging subsystem and the Kalman update only if have valid observations
valid_times = [z for z in obs_data.keys() if abs(total_seconds(z - model_time)) < assim_time_win/2.0]
print('INFO: there are %d valid times at model time %s for fuel index %d' % (len(valid_times), str(model_time), fuel_ndx))
if len(valid_times) > 0:
# add model time as time when assimilation occurred
diagnostics().push("mta", model_time)
# retrieve observations for current time
obs_valid_now = []
for z in valid_times:
obs_valid_now.extend(obs_data[z])
print('INFO: model time %s, assimilating %d observations.' % (str(model_time), len(obs_valid_now)))
# construct covariates for this time instant
X[:,:,0] = f[:,:,fuel_ndx]
for i in range(1, Xd3):
cov_id = cov_ids[i-1]
if cov_id in static_covar_map:
X[:, :, i] = Xr[:, :, i]
elif cov_id in dynamic_covar_map:
F = dynamic_covar_map[cov_id]
X[:, :, i] = F[t, :, :]
else:
error('FATAL: found unknown covariate %s' % cov_id)
# find differences (residuals) between observed measurements and nearest grid points
obs_vals = [o.get_value() for o in obs_valid_now]
obs_ngp = [o.get_nearest_grid_point() for o in obs_valid_now]
diagnostics().push("obs_vals", obs_vals)
diagnostics().push("obs_ngp", obs_ngp)
mod_vals = np.array([f[i,j,fuel_ndx] for i,j in obs_ngp])
mod_na_vals = np.array([f_na[i,j,fuel_ndx] for i,j in obs_ngp])
diagnostics().push("fm10f_rmse", np.mean((obs_vals - mod_vals)**2)**0.5)
diagnostics().push("fm10na_rmse", np.mean((obs_vals - mod_na_vals)**2)**0.5)
# krige observations to grid points
Kf_fn, Vf_fn = fit_tsm(obs_valid_now, X)
if np.count_nonzero(Kf_fn > 2.5) > 0:
rain_t = dynamic_covar_map['rain'][t,:,:]
print('WARN: in TSM found %d values over 2.5, %d of those had rain, clamped to 2.5' %
(np.count_nonzero(Kf_fn > 2.5),
np.count_nonzero(np.logical_and(Kf_fn > 2.5, rain_t > 0.0))))
Kf_fn[Kf_fn > 2.5] = 2.5
if np.count_nonzero(Kf_fn < 0.0) > 0:
print('WARN: in TSM found %d values under 0.0, clamped to 0.0' % np.count_nonzero(Kf_fn < 0.0))
Kf_fn[Kf_fn < 0.0] = 0.0
krig_vals = np.array([Kf_fn[ngp] for ngp in obs_ngp])
diagnostics().push("assim_info", (t, fuel_ndx, obs_vals, krig_vals, mod_vals, mod_na_vals))
diagnostics().push("fm10_kriging_var", (t, np.mean(Vf_fn)))
if cfg['write_fields'] == 'all':
nctsmV[t,:,:] = Vf_fn
# append to storage for kriged fields in this time instant
Kfs.append(Kf_fn)
Vfs.append(Vf_fn)
fns.append(fuel_ndx)
# if there were any observations, run the kalman update step
if len(fns) > 0:
NobsClasses = len(fns)
O = np.zeros((dom_shape[0], dom_shape[1], NobsClasses))
V = np.zeros((dom_shape[0], dom_shape[1], NobsClasses, NobsClasses))
for i in range(NobsClasses):
O[:,:,i] = Kfs[i]
V[:,:,i,i] = Vfs[i]
# execute the Kalman update
if len(fns) == 1:
models.kalman_update_single2(O, V, fns[0], Kg)
else:
models.kalman_update(O, V, fns, Kg)
# push new diagnostic outputs
if cfg['write_fields'] == 'all':
ncKg[t,:,:] = Kg[:,:,1]
for i in range(4):
diagnostics().push("K%d_summary" % i, (t, np.amin(Kg[:,:,i]), np.mean(Kg[:,:,i]), np.amax(Kg[:,:,i])))
if np.any(models.get_state()[:,:,i] < 0.0):
print("WARN: in field %d there were %d negative moisture values !" % (i, np.count_nonzero(models.get_state()[:,:,i] < 0.0)))
ind = np.unravel_index(np.argmin(models.get_state()[:,:,i]), models.get_state().shape[:2])
print(models.P[ind[0],ind[1],:,:])
print("TSM input at given position: value %g variance %g" % (O[ind[0],ind[1]], V[ind[0],ind[1]]))
print("Model state at given position:")
print(models.m_ext[ind[0],ind[1],:])
# store post-assimilation (or forecast depending on whether observations were available) FM-10 state and variance
if cfg['write_fields'] == 'all':
ncfm10a[t,:,:] = models.get_state()[:,:,1]
ncfm10aV[t,:,:] = models.P[:,:,1,1]
# we don't care if we assimilated or not, we always check our error on target station if in test mode
if test_mode:
valid_times = [z for z in tgt_obs_fm10.keys() if abs(total_seconds(z - model_time)) < assim_time_win/2.0]
tgt_i, tgt_j = test_ngp
diagnostics().push("test_pred", f[tgt_i, tgt_j,1])
if len(valid_times) > 0:
# this is our target observation [FIXME: this disregards multiple observations if multiple happen to be valid]
tgt_obs = tgt_obs_fm10[valid_times[0]][0]
obs = tgt_obs.get_value()
diagnostics().push("test_obs", obs)
else:
diagnostics().push("test_obs", np.nan)
# store data in wrf_file variable FMC_G
if cfg['write_fields']:
ncfmc_gc[t,:Nk,:,:] = np.transpose(models.get_state()[:,:,:Nk],axes=[2,0,1])
# store the diagnostics in a binary file when done
diagnostics().dump_store(os.path.join(cfg['output_dir'], 'diagnostics.bin'))
# close the netCDF file (relevant if we did write into FMC_GC)
if out_file is not None:
out_file.close()
def run_module():
# read in configuration file to execute run
print("Reading configuration from [%s]" % sys.argv[1])
with open(sys.argv[1]) as f:
cfg = eval(f.read())
# ensure output path exists
if not os.path.isdir(cfg['output_dir']):
os.mkdir(cfg['output_dir'])
# configure diagnostics
init_diagnostics(
os.path.join(cfg['output_dir'], 'moisture_model_v1_diagnostics.txt'))
# Trend surface model diagnostics
diagnostics().configure_tag("kriging_cov_cond", True, True, True)
diagnostics().configure_tag("s2_eta_hat", True, True, True)
diagnostics().configure_tag("kriging_rmse", True, True, True)
diagnostics().configure_tag("kriging_beta", True, True, True)
diagnostics().configure_tag("kriging_iters", False, True, True)
diagnostics().configure_tag("kriging_subzero_s2_estimates", False, True,
True)
diagnostics().configure_tag("fm10_kriging_var", True, True, True)
diagnostics().configure_tag("f0_summary", True, True, True)
diagnostics().configure_tag("f1_summary", True, True, True)
diagnostics().configure_tag("f2_summary", True, True, True)
diagnostics().configure_tag("f3_summary", True, True, True)
# Assimilation parameters
diagnostics().configure_tag("K0_summary", True, True, True)
diagnostics().configure_tag("K1_summary", True, True, True)
diagnostics().configure_tag("K2_summary", True, True, True)
diagnostics().configure_tag("K3_summary", True, True, True)
diagnostics().configure_tag("assim_info", False, False, True)
# Model forecast, analysis and non-assimilated model: state, covariance, errors
diagnostics().configure_tag("fm10f_rmse", True, True, True)
diagnostics().configure_tag("fm10na_rmse", True, True, True)
# all simulation times and all assimilation times (subset)
diagnostics().configure_tag("mta", False, True, True)
diagnostics().configure_tag("mt", False, True, True)
# observation values and their nearest grid points
diagnostics().configure_tag("obs_vals", False, True, True)
diagnostics().configure_tag("obs_ngp", False, True, True)
# in test mode, we will emit observations at the target station
# our predictions, the nearest grid point and the test station id
diagnostics().configure_tag("test_obs", True, True, True)
diagnostics().configure_tag("test_pred", True, True, True)
diagnostics().configure_tag("test_ngp", True, True, True)
diagnostics().configure_tag("test_station_id", True, True, True)
### Load and preprocess WRF model data
# load WRF data
wrf_data = WRFModelData(cfg['wrf_output'],
['T2', 'Q2', 'PSFC', 'RAINNC', 'RAINC', 'HGT'])
wrf_data.slice_field('HGT')
# read in spatial and temporal extent of WRF variables
lat, lon = wrf_data.get_lats(), wrf_data.get_lons()
hgt = wrf_data['HGT']
tm = wrf_data.get_gmt_times()
Nt = cfg['Nt'] if cfg.has_key('Nt') and cfg['Nt'] is not None else len(tm)
dom_shape = lat.shape
print('INFO: domain size is %d x %d grid points.' % dom_shape)
print('INFO: domain extent is lats (%g to %g) lons (%g to %g).' %
(np.amin(lat), np.amax(lat), np.amin(lon), np.amax(lon)))
# if writing is requested, open output file and set up dimensions
if cfg['write_fields'] not in ['all', 'fmc_gc', 'none']:
error('FATAL: write_fields must be one of all, fmc_gc or none.')
if cfg['write_fields'] == 'none':
cfg['write_fields'] = False
out_file = None
ncfmc_gc, ncfm10a, ncfm10aV, ncfm10f, cnfm10fV, ncfm10na = None, None, None, None, None, None
nctsmV, ncKg = None, None
if cfg['write_fields']:
out_file = netCDF4.Dataset(cfg['output_dir'] + '/fields.nc', 'w')
out_file.createDimension('Time', None)
out_file.createDimension('fuel_moisture_classes_stag', 5)
out_file.createDimension('south_north', dom_shape[0])
out_file.createDimension('west_east', dom_shape[1])
ncfmc_gc = out_file.createVariable(
'FMC_GC', 'f4',
('Time', 'fuel_moisture_classes_stag', 'south_north', 'west_east'))
if cfg['write_fields'] == 'all':
ncfm10a = out_file.createVariable(
'fm10a', 'f4', ('Time', 'south_north', 'west_east'))
ncfm10aV = out_file.createVariable(
'fm10a_var', 'f4', ('Time', 'south_north', 'west_east'))
ncfm10na = out_file.createVariable(
'fm10na', 'f4', ('Time', 'south_north', 'west_east'))
ncfm10f = out_file.createVariable(
'fm10f', 'f4', ('Time', 'south_north', 'west_east'))
ncfm10fV = out_file.createVariable(
'fm10f_var', 'f4', ('Time', 'south_north', 'west_east'))
nctsmV = out_file.createVariable(
'tsm_var', 'f4', ('Time', 'south_north', 'west_east'))
ncKg = out_file.createVariable(
'kalman_gain', 'f4', ('Time', 'south_north', 'west_east'))
print('INFO: opened fields.nc for writing ALL output fields.')
else:
print('INFO: opened field.nc for writing FMC_GC only.')
test_mode = (cfg['run_mode'] == 'test')
tgt_station = None
if cfg['run_mode'] == 'test':
print('INFO: running in TEST mode! Will perform leave-one-out tesing.')
tgt_station_id = cfg['target_station_id']
diagnostics().push('test_station_id', tgt_station_id)
elif cfg['run_mode'] == 'production':
print(
'INFO: running in PRODUCTION mode! Using all observation stations.'
)
else:
error('FATAL: invalid run mode! Must be "test" or "production".')
# determine simulation times
tm_start = parse_datetime(
cfg['start_time']) if cfg['start_time'] is not None else tm[0]
tm_end = parse_datetime(
cfg['end_time']) if cfg['end_time'] is not None else tm[-1]
# if the required start time or end time are outside the simulation domain, exit with an error
if tm_start < tm[0] or tm_end > tm[-1]:
print('FATAL: invalid time range, required [%s-%s], availble [%s-%s]' %
(str(tm_start), str(tm_end), str(tm[0]), str(tm[-1])))
sys.exit(2)
print('INFO: time limits are %s to %s\nINFO: simulation is from %s to %s' %
(str(tm_start), str(tm_end), str(tm[0]), str(tm[-1])))
# retrieve dynamic covariates and remove mean at each time point for T2 and PSFC
T2 = wrf_data['T2']
#T2 -= np.mean(np.mean(T2,axis=0),axis=0)[np.newaxis,np.newaxis,:]
PSFC = wrf_data['PSFC']
#PSFC -= np.mean(np.mean(PSFC,axis=0),axis=0)[np.newaxis,np.newaxis,:]
# numerical fix - if it rains at an intensity of less than 0.001 per hour, set rain to zero
# also, use log(rain + 1) to prevent wild trend surface model predictions when stations see little rain
# but elsewhere there is too much rain
# without this, numerical errors in trend surface model may pop up
rain = wrf_data['RAIN']
#rain[rain < 0.01] = 0.0
rain = np.log(rain + 1.0)
# moisture equilibria are now computed from averaged Q,P,T at beginning and end of period
Ed, Ew = wrf_data.get_moisture_equilibria()
### Load observation data from the stations
# compute the diagonal distance between grid points
grid_dist_km = great_circle_distance(lon[0, 0], lat[0, 0], lon[1, 1],
lat[1, 1])
print('INFO: diagonal distance in grid is %g' % grid_dist_km)
# load station data from files
with open(cfg['station_list_file'], 'r') as f:
si_list = f.read().split('\n')
si_list = filter(lambda x: len(x) > 0 and x[0] != '#',
map(string.strip, si_list))
# for each station id, load the station
stations = []
for code in si_list:
mws = MesoWestStation(code)
mws.load_station_info(
os.path.join(cfg["station_info_dir"], "%s.info" % code))
mws.register_to_grid(wrf_data)
if mws.get_dist_to_grid() < grid_dist_km / 2.0:
print(
'Station %s: lat %g lon %g nearest grid pt %s lat %g lon %g dist_to_grid %g'
% (code, mws.lat, mws.lon, str(mws.grid_pt), lat[mws.grid_pt],
lon[mws.grid_pt], mws.dist_grid_pt))
mws.load_station_data(
os.path.join(cfg["station_data_dir"], "%s.obs" % code))
if test_mode and mws.get_id() == tgt_station_id:
tgt_station = mws
print(
'INFO: in test mode, targeting station %s (removed from data pool).'
% tgt_station_id)
diagnostics().push("test_ngp", mws.get_nearest_grid_point())
else:
stations.append(mws)
print('Loaded %d stations (discarded %d stations, too far from grid).' %
(len(stations), len(si_list) - len(stations)))
if test_mode and tgt_station is None:
error(
'FATAL: in test mode, a station was removed that was not among accepted stations.'
)
# build the observation data
obs_data_fm10 = build_observation_data(stations, 'FM')
# build target data if in test mode
tgt_obs_fm10 = None
test_ngp = None
if test_mode:
test_ngp = tgt_station.get_nearest_grid_point()
tgt_obs_fm10 = build_observation_data([tgt_station], 'FM')
### Initialize model and visualization
# construct initial conditions from timestep 0
E = 0.5 * (Ed[0, :, :] + Ew[0, :, :])
# set up parameters
Nk = 4 # we simulate 4 types of fuel
Q = np.diag(cfg['Q'])
P0 = np.diag(cfg['P0'])
Tk = np.array([1.0, 10.0, 100.0, 1000.0]) * 3600
dt = (tm[1] - tm[0]).seconds
print("INFO: Computed timestep from WRF is is %g seconds." % dt)
mresV = np.zeros_like(E)
mid = np.zeros_like(E)
Kg = np.zeros((dom_shape[0], dom_shape[1], len(Tk) + 2))
# preprocess all static covariates
cov_ids = cfg['covariates']
Xd3 = len(cov_ids) + 1
X = np.zeros((dom_shape[0], dom_shape[1], Xd3))
Xr = np.zeros((dom_shape[0], dom_shape[1], Xd3))
static_covar_map = {
"lon": lon - np.mean(lon),
"lat": lat - np.mean(lat),
"elevation": hgt - np.mean(hgt),
"constant": np.ones(dom_shape)
}
dynamic_covar_map = {"temperature": T2, "pressure": PSFC, "rain": rain}
for i in range(1, Xd3):
cov_id = cov_ids[i - 1]
if cov_id in static_covar_map:
print('INFO: found static covariate %s' % cov_id)
Xr[:, :, i] = static_covar_map[cov_id]
elif cov_id in dynamic_covar_map:
print('INFO: found dynamic covariate %s' % cov_id)
else:
print('FATAL: unknown covariate %s encountered' % cov_id)
sys.exit(2)
print("INFO: there are %d covariates (including model state)" % Xd3)
# retrieve assimilation time window
assim_time_win = cfg['assimilation_time_window']
print('GMM init: equilibrium (%g,%g,%g) and at 86,205 %g' %
(np.amin(E), np.mean(E), np.amax(E), E[86, 205]))
models = GridMoistureModel(
E[:, :, np.newaxis][:, :, np.zeros((4, ), dtype=np.int)], Tk, P0)
models_na = GridMoistureModel(
E[:, :, np.newaxis][:, :, np.zeros((4, ), dtype=np.int)], Tk, P0)
### Run model for each WRF timestep and assimilate data when available
t_start, t_end = 1, len(tm) - 1
while tm_start > tm[t_start]:
t_start += 1
while tm_end < tm[t_end]:
t_end -= 1
# the first FMC_GC value gets filled out with equilibria
if cfg['write_fields']:
for i in range(Nk):
ncfmc_gc[0, i, :, :] = E
print('INFO: running simulation from %s (%d) to %s (%d).' %
(str(tm[t_start]), t_start, str(tm[t_end]), t_end))
for t in range(t_start, t_end + 1):
model_time = tm[t]
print("INFO: time: %s, step: %d" % (str(model_time), t))
diagnostics().push("mt", model_time)
models_na.advance_model(Ed[t - 1, :, :], Ew[t - 1, :, :],
rain[t - 1, :, :], dt, Q)
models.advance_model(Ed[t - 1, :, :], Ew[t - 1, :, :],
rain[t - 1, :, :], dt, Q)
# extract fuel moisture contents [make a fresh copy every iteration!]
f = models.get_state().copy()
f_na = models_na.get_state().copy()
# push 10-hr fuel state & variance of forecast
if cfg['write_fields'] == 'all':
ncfm10f[t, :, :] = models.get_state()[:, :, 1]
ncfm10fV[t, :, :] = models.P[:, :, 1, 1]
ncfm10na[t, :, :] = models_na.get_state()[:, :, 1]
# examine the assimilated fields (if assimilation is activated)
for i in range(4):
diagnostics().push("f%d_summary" % i, (t, np.amin(
f[:, :, i]), np.mean(f[:, :, i]), np.amax(f[:, :, i])))
if np.any(f[:, :, i] < 0.0):
print(
"WARN: in field %d there were %d negative moisture values !"
% (i, np.count_nonzero(f[:, :, i] < 0.0)))
ind = np.unravel_index(np.argmin(f[:, :, i]), f.shape[:2])
print(models.P[ind[0], ind[1], :, :])
print("Full model state at position %d,%d:" % (ind[0], ind[1]))
print(models.m_ext[ind[0], ind[1], :])
if np.any(f[:, :, i] > 2.5):
print(
"WARN: in field %d there were %d moisture values above 2.5!"
% (i, np.count_nonzero(f[:, :, i] > 2.5)))
ind = np.unravel_index(np.argmax(f[:, :, i]), f.shape[:2])
print(models.P[ind[0], ind[1], :, :])
print("Full model state at position %d,%d:" % (ind[0], ind[1]))
print(models.m_ext[ind[0], ind[1], :])
if cfg['assimilate']:
# run Kriging on each observed fuel type
Kfs, Vfs, fns = [], [], []
for obs_data, fuel_ndx in [(obs_data_fm10, 1)]:
# run the kriging subsystem and the Kalman update only if have valid observations
valid_times = [
z for z in obs_data.keys()
if abs(total_seconds(z - model_time)) < assim_time_win /
2.0
]
print(
'INFO: there are %d valid times at model time %s for fuel index %d'
% (len(valid_times), str(model_time), fuel_ndx))
if len(valid_times) > 0:
# add model time as time when assimilation occurred
diagnostics().push("mta", model_time)
# retrieve observations for current time
obs_valid_now = []
for z in valid_times:
obs_valid_now.extend(obs_data[z])
print(
'INFO: model time %s, assimilating %d observations.' %
(str(model_time), len(obs_valid_now)))
# construct covariates for this time instant
X[:, :, 0] = f[:, :, fuel_ndx]
for i in range(1, Xd3):
cov_id = cov_ids[i - 1]
if cov_id in static_covar_map:
X[:, :, i] = Xr[:, :, i]
elif cov_id in dynamic_covar_map:
F = dynamic_covar_map[cov_id]
X[:, :, i] = F[t, :, :]
else:
error('FATAL: found unknown covariate %s' % cov_id)
# find differences (residuals) between observed measurements and nearest grid points
obs_vals = [o.get_value() for o in obs_valid_now]
obs_ngp = [
o.get_nearest_grid_point() for o in obs_valid_now
]
diagnostics().push("obs_vals", obs_vals)
diagnostics().push("obs_ngp", obs_ngp)
mod_vals = np.array(
[f[i, j, fuel_ndx] for i, j in obs_ngp])
mod_na_vals = np.array(
[f_na[i, j, fuel_ndx] for i, j in obs_ngp])
diagnostics().push("fm10f_rmse",
np.mean((obs_vals - mod_vals)**2)**0.5)
diagnostics().push(
"fm10na_rmse",
np.mean((obs_vals - mod_na_vals)**2)**0.5)
# krige observations to grid points
Kf_fn, Vf_fn = fit_tsm(obs_valid_now, X)
if np.count_nonzero(Kf_fn > 2.5) > 0:
rain_t = dynamic_covar_map['rain'][t, :, :]
print(
'WARN: in TSM found %d values over 2.5, %d of those had rain, clamped to 2.5'
% (np.count_nonzero(Kf_fn > 2.5),
np.count_nonzero(
np.logical_and(Kf_fn > 2.5, rain_t > 0.0))))
Kf_fn[Kf_fn > 2.5] = 2.5
if np.count_nonzero(Kf_fn < 0.0) > 0:
print(
'WARN: in TSM found %d values under 0.0, clamped to 0.0'
% np.count_nonzero(Kf_fn < 0.0))
Kf_fn[Kf_fn < 0.0] = 0.0
krig_vals = np.array([Kf_fn[ngp] for ngp in obs_ngp])
diagnostics().push("assim_info",
(t, fuel_ndx, obs_vals, krig_vals,
mod_vals, mod_na_vals))
diagnostics().push("fm10_kriging_var", (t, np.mean(Vf_fn)))
if cfg['write_fields'] == 'all':
nctsmV[t, :, :] = Vf_fn
# append to storage for kriged fields in this time instant
Kfs.append(Kf_fn)
Vfs.append(Vf_fn)
fns.append(fuel_ndx)
# if there were any observations, run the kalman update step
if len(fns) > 0:
NobsClasses = len(fns)
O = np.zeros((dom_shape[0], dom_shape[1], NobsClasses))
V = np.zeros(
(dom_shape[0], dom_shape[1], NobsClasses, NobsClasses))
for i in range(NobsClasses):
O[:, :, i] = Kfs[i]
V[:, :, i, i] = Vfs[i]
# execute the Kalman update
if len(fns) == 1:
models.kalman_update_single2(O, V, fns[0], Kg)
else:
models.kalman_update(O, V, fns, Kg)
# push new diagnostic outputs
if cfg['write_fields'] == 'all':
ncKg[t, :, :] = Kg[:, :, 1]
for i in range(4):
diagnostics().push(
"K%d_summary" % i,
(t, np.amin(Kg[:, :, i]), np.mean(
Kg[:, :, i]), np.amax(Kg[:, :, i])))
if np.any(models.get_state()[:, :, i] < 0.0):
print(
"WARN: in field %d there were %d negative moisture values !"
% (i,
np.count_nonzero(
models.get_state()[:, :, i] < 0.0)))
ind = np.unravel_index(
np.argmin(models.get_state()[:, :, i]),
models.get_state().shape[:2])
print(models.P[ind[0], ind[1], :, :])
print(
"TSM input at given position: value %g variance %g"
% (O[ind[0], ind[1]], V[ind[0], ind[1]]))
print("Model state at given position:")
print(models.m_ext[ind[0], ind[1], :])
# store post-assimilation (or forecast depending on whether observations were available) FM-10 state and variance
if cfg['write_fields'] == 'all':
ncfm10a[t, :, :] = models.get_state()[:, :, 1]
ncfm10aV[t, :, :] = models.P[:, :, 1, 1]
# we don't care if we assimilated or not, we always check our error on target station if in test mode
if test_mode:
valid_times = [
z for z in tgt_obs_fm10.keys()
if abs(total_seconds(z - model_time)) < assim_time_win /
2.0
]
tgt_i, tgt_j = test_ngp
diagnostics().push("test_pred", f[tgt_i, tgt_j, 1])
if len(valid_times) > 0:
# this is our target observation [FIXME: this disregards multiple observations if multiple happen to be valid]
tgt_obs = tgt_obs_fm10[valid_times[0]][0]
obs = tgt_obs.get_value()
diagnostics().push("test_obs", obs)
else:
diagnostics().push("test_obs", np.nan)
# store data in wrf_file variable FMC_G
if cfg['write_fields']:
ncfmc_gc[t, :Nk, :, :] = np.transpose(
models.get_state()[:, :, :Nk], axes=[2, 0, 1])
# store the diagnostics in a binary file when done
diagnostics().dump_store(os.path.join(cfg['output_dir'],
'diagnostics.bin'))
# close the netCDF file (relevant if we did write into FMC_GC)
if out_file is not None:
out_file.close()
def universal_kriging_data_to_model(obs_data, obs_stds, m, wrf_data, mod_stds,
t):
"""
Universal kriging of data points to model points. The kriging results in
the matrix K, which contains the kriged observations and the matrix V,
which contains the kriging variance.
"""
mlons, mlats = wrf_data.get_lons(), wrf_data.get_lats()
K = np.zeros_like(mlons)
V = np.zeros_like(mlons)
Nobs = len(obs_data)
obs_vals = np.zeros((Nobs, ))
station_lonlat = []
gridndx = []
mu_obs = np.zeros((Nobs, ))
m_pred = np.zeros((Nobs, ))
measV = np.zeros((Nobs, ))
# accumulate the indices of the nearest grid points
ndx = 0
for obs in obs_data:
gridndx.append(obs.get_nearest_grid_point())
obs_vals[ndx] = obs.get_value()
station_lonlat.append(obs.get_position())
measV[ndx] = obs.get_measurement_variance()
m_pred[ndx] = m[obs.get_nearest_grid_point()]
ndx += 1
# convert lists to column vectors
m_pred = np.asmatrix(m_pred).T
obs_vals = np.asmatrix(obs_vals).T
# construct the covariance matrix and invert it
C = np.asmatrix(construct_spatial_correlation_matrix2(station_lonlat))
oS = np.asmatrix(np.diag(obs_stds))
Sigma = oS.T * C * oS + np.diag(measV)
SigInv = np.linalg.inv(Sigma)
# estimate the fixed part of the model
gamma = np.linalg.inv(
m_pred.T * SigInv * m_pred) * m_pred.T * SigInv * obs_vals
gamma = gamma[0, 0]
mu_mod = m * gamma
ndx = 0
for obs in obs_data:
mu_obs[ndx] = mu_mod[obs.get_nearest_grid_point()]
ndx += 1
# compute observation residuals (using model fit from examine_station_data)
# and the mean absolute deviation
res_obs = np.asmatrix(obs_vals - mu_obs).T
mape = np.mean(np.abs(res_obs))
diagnostics().push("skdm_cov_cond", np.linalg.cond(Sigma))
# precompute some matrices
xs = obs_vals.T * SigInv
xsx_1 = 1.0 / (xs * obs_vals)
# run the kriging estimator for each model grid point
K = np.zeros_like(mlats)
cov = np.asmatrix(np.zeros_like(mu_obs)).T
for p in np.ndindex(K.shape):
# compute the covariance array anew for each grid point
for k in range(Nobs):
lon, lat = station_lonlat[k]
cc = max(
0.8565 -
0.0063 * great_circle_distance(mlons[p], mlats[p], lon, lat),
0.0)
cov[k, 0] = mod_stds[p] * cc * obs_stds[k]
csi = SigInv * (cov - obs_vals * xsx_1 * (xs * cov - mu_mod[p]))
K[p] = csi.T * obs_vals
tmp = (mu_mod[p] - cov.T * SigInv * obs_vals)
V[p] = mod_stds[p]**2 - cov.T * SigInv * cov + tmp * xsx_1 * tmp
return K, V, gamma, mape
if len(obs_at_t) > 0:
# construct pairwise dataset
dists = []
sqdiffs = []
for i in range(len(obs_at_t)):
sid = obs_at_t[i].get_station().get_id()
pi = obs_at_t[i].get_position()
stats = station_stats[sid]
oi = obs_at_t[i].get_value()
if cfg['standardize']:
oi = (oi - stats[0]) / stats[1]
for j in range(i + 1, len(obs_at_t)):
pj = obs_at_t[j].get_position()
oj = obs_at_t[j].get_value()
dist = great_circle_distance(pi[0], pi[1], pj[0],
pj[1])
if dist < max_dist:
dists.append(dist)
sqdiffs.append(0.5 * (oi - oj)**2)
all_dists.append(dist)
all_sqdiffs.append(0.5 * (oi - oj)**2)
fname = 'std_variogram_est_%03d.png' % t if cfg[
'standardize'] else 'variogram_est_%03d.png' % t
plot_variogram(dists, sqdiffs, bins,
'Variogram at time %s' % str(t_now),
os.path.join(cfg['output_dir'], fname))
fname = 'std_variogram_est_all.png' if cfg[
'standardize'] else 'variogram_est_all.png'
plot_variogram(all_dists, all_sqdiffs, bins,
si_list = f.read().split('\n')
si_list = filter(lambda x: (len(x) > 0) and (x[0] != '#'), map(string.strip, si_list))
stations = {}
for code in si_list:
mws = MesoWestStation(code)
mws.load_station_info(os.path.join(cfg["station_info_dir"], "%s.info" % code))
stations[code] = mws
# first construct a distance matrix for all stations
st_dists = np.zeros((len(sids_list), len(sids_list)))
for j in range(len(sids_list)):
lonj, latj = stations[sids_list[j]].get_position()
for k in range(j+1, len(sids_list)):
lonk, latk = stations[sids_list[k]].get_position()
d = great_circle_distance(lonj, latj, lonk, latk)
st_dists[j, k] = d
st_dists[k, j] = d
# accumulate data over all time points
for i in range(N):
dists_i = []
sqdiffs_i = []
# retrieve valid measurements, rest is nan
di = data[i]
dobs = di['kriging_obs']
Nobs = len(dobs)
obs = np.zeros(len(sids_list))
def universal_kriging_data_to_model(obs_data, obs_stds, m, wrf_data, mod_stds, t):
"""
Universal kriging of data points to model points. The kriging results in
the matrix K, which contains the kriged observations and the matrix V,
which contains the kriging variance.
"""
mlons, mlats = wrf_data.get_lons(), wrf_data.get_lats()
K = np.zeros_like(mlons)
V = np.zeros_like(mlons)
Nobs = len(obs_data)
obs_vals = np.zeros((Nobs,))
station_lonlat = []
gridndx = []
mu_obs = np.zeros((Nobs,))
m_pred = np.zeros((Nobs,))
measV = np.zeros((Nobs,))
# accumulate the indices of the nearest grid points
ndx = 0
for obs in obs_data:
gridndx.append(obs.get_nearest_grid_point())
obs_vals[ndx] = obs.get_value()
station_lonlat.append(obs.get_position())
measV[ndx] = obs.get_measurement_variance()
m_pred[ndx] = m[obs.get_nearest_grid_point()]
ndx += 1
# convert lists to column vectors
m_pred = np.asmatrix(m_pred).T
obs_vals = np.asmatrix(obs_vals).T
# construct the covariance matrix and invert it
C = np.asmatrix(construct_spatial_correlation_matrix2(station_lonlat))
oS = np.asmatrix(np.diag(obs_stds))
Sigma = oS.T * C * oS + np.diag(measV)
SigInv = np.linalg.inv(Sigma)
# estimate the fixed part of the model
gamma = np.linalg.inv(m_pred.T * SigInv * m_pred) * m_pred.T * SigInv * obs_vals
gamma = gamma[0, 0]
mu_mod = m * gamma
ndx = 0
for obs in obs_data:
mu_obs[ndx] = mu_mod[obs.get_nearest_grid_point()]
ndx += 1
# compute observation residuals (using model fit from examine_station_data)
# and the mean absolute deviation
res_obs = np.asmatrix(obs_vals - mu_obs).T
mape = np.mean(np.abs(res_obs))
diagnostics().push("skdm_cov_cond", np.linalg.cond(Sigma))
# precompute some matrices
xs = obs_vals.T * SigInv
xsx_1 = 1.0 / (xs * obs_vals)
# run the kriging estimator for each model grid point
K = np.zeros_like(mlats)
cov = np.asmatrix(np.zeros_like(mu_obs)).T
for p in np.ndindex(K.shape):
# compute the covariance array anew for each grid point
for k in range(Nobs):
lon, lat = station_lonlat[k]
cc = max(0.8565 - 0.0063 * great_circle_distance(mlons[p], mlats[p], lon, lat), 0.0)
cov[k, 0] = mod_stds[p] * cc * obs_stds[k]
csi = SigInv * (cov - obs_vals * xsx_1 * (xs * cov - mu_mod[p]))
K[p] = csi.T * obs_vals
tmp = mu_mod[p] - cov.T * SigInv * obs_vals
V[p] = mod_stds[p] ** 2 - cov.T * SigInv * cov + tmp * xsx_1 * tmp
return K, V, gamma, mape
with open(cfg['station_list_file'], 'r') as f:
si_list = f.read().split('\n')
si_list = filter(lambda x: len(x) > 0 and x[0] != '#',
map(string.strip, si_list))
# for each station id, load the station
stations = []
for code in si_list:
mws = MesoWestStation(code)
mws.load_station_info(
os.path.join(cfg["station_info_dir"], "%s.info" % code))
stations.append(mws)
cmd = sys.argv[2]
if cmd == 'find_closest':
lat = float(sys.argv[3])
lon = float(sys.argv[4])
min_dist = 1000000 # that should be more [km] than anything we will find :)
closest = None
for st in stations:
slon, slat = st.get_position()
d = great_circle_distance(lon, lat, slon, slat)
if d < min_dist:
closest = st
min_dist = d
clon, clat = closest.get_position()
print('Closest station is %s at %g,%g with dist %g km' %
(closest.get_id(), clat, clon, min_dist))
with open(cfg['station_list_file'], 'r') as f:
si_list = f.read().split('\n')
si_list = filter(lambda x: len(x) > 0 and x[0] != '#', map(string.strip, si_list))
# for each station id, load the station
stations = []
for code in si_list:
mws = MesoWestStation(code)
mws.load_station_info(os.path.join(cfg["station_info_dir"], "%s.info" % code))
stations.append(mws)
cmd = sys.argv[2]
if cmd == 'find_closest':
lat = float(sys.argv[3])
lon = float(sys.argv[4])
min_dist = 1000000 # that should be more [km] than anything we will find :)
closest = None
for st in stations:
slon,slat = st.get_position()
d = great_circle_distance(lon,lat,slon,slat)
if d < min_dist:
closest = st
min_dist = d
clon,clat = closest.get_position()
print('Closest station is %s at %g,%g with dist %g km' % (closest.get_id(), clat, clon, min_dist))
def run_module():
# read in configuration file to execute run
print("Reading configuration from [%s]" % sys.argv[1])
with open(sys.argv[1]) as f:
cfg = eval(f.read())
# init diagnostics
init_diagnostics(os.path.join(cfg['output_dir'], 'moisture_model_v1_diagnostics.txt'))
diagnostics().configure_tag("s2_eta_hat", True, True, True)
diagnostics().configure_tag("kriging_rmse", True, True, True)
diagnostics().configure_tag("kriging_beta", True, True, True)
diagnostics().configure_tag("kriging_iters", False, True, True)
diagnostics().configure_tag("kriging_subzero_s2_estimates", False, True, True)
# load the wrfinput file
wrfin = WRFModelData(cfg['wrf_input'], ['T2', 'Q2', 'PSFC', 'HGT', 'FMC_GC', 'FMEP'])
lat, lon = wrfin.get_lats(), wrfin.get_lons()
ts_now = wrfin['GMT'][0]
dom_shape = lat.shape
print('INFO: domain size is %d x %d grid points, wrfinput timestamp %s' % (dom_shape[0], dom_shape[1], str(ts_now)))
print('INFO: domain extent is lats (%g to %g) lons (%g to %g).' % (np.amin(lat),np.amax(lat),np.amin(lon),np.amax(lon)))
# compute the diagonal distance between grid points
grid_dist_km = great_circle_distance(lon[0,0], lat[0,0], lon[1,1], lat[1,1])
print('INFO: diagonal distance in grid is %g' % grid_dist_km)
# load observations but discard those too far away from the grid nodes
obss = load_raws_observations(cfg['observations'], lat, lon, grid_dist_km)
fm10 = build_observation_data(obss)
print('INFO: %d different time instances found in observations' % len(fm10))
# if a previous cycle is available (i.e. the wrfoutput is a valid file)
if os.path.exists(cfg['wrf_output_prev']) and check_overlap(cfg['wrf_output_prev'],ts_now):
# load the model as a wrfout with all default variables
wrfout = WRFModelData(cfg['wrf_output_prev'])
outts = wrfout['GMT']
print("INFO: previous forecast [%s - %s] exists, running DA till %s" % (str(outts[0]),str(outts[-1]),str(ts_now)))
# run from the start until now (retrieve fuel moisture, extended parameters, covariance matrix)
model = run_data_assimilation(wrfout, fm10, ts_now, cfg)
# store this for the current time instance (fm, ep in the wrfinput, P next to it)
d = netCDF4.Dataset(cfg['wrf_input'], 'r+')
d.variables['FMC_GC'] = fm
d.variables['FMEP'] = ep
d.close()
# store the covariance matrix alongside the wrfinput file
dir = os.path.dirname(wrfin)
store_covariance_matrix(P, os.path.join(dir, 'P.nc'))
else:
print("INFO: no previous forecast found, running DA from equilibrium at %s" % (str(ts_now)))
# initialize from weather equilibrium and perform one DA step
model = init_from_equilibrium(wrfin, fm10, ts_now, cfg)
# store result in wrfinput dataset
d = netCDF4.Dataset(cfg['wrf_input'], 'r+')
fmcep = model.get_state()
d.variables['FMC_GC'][0,:3,:,:] = fmcep[:,:,:3].transpose((2,0,1))
d.variables['FMEP'][0,:,:,:] = fmcep[:,:,3:5].transpose((2,0,1))
d.close()
store_covariance_matrix(model.get_state_covar(), os.path.join(os.path.dirname(cfg['wrf_input']), 'P.nc'))
return 0
