def execute_da_step(model, model_time, covariates, fm10):
"""
Execute a single DA step from the current state/extended parameters and covariance matrix using
the <covariates> and observations <fm10>. Assimilation time window is fixed at 60 mins.
:param model: a FuelMoistureModel
:param model_time: the current model time
:param covariates: the covariate fields to take into account to model the spatial structure of the FM field
:param fm10: the 10-hr fuel moisture observations
"""
valid_times = [
z for z in fm10.keys() if abs((z - model_time).total_seconds()) < 1800
]
if len(valid_times) > 0:
# retrieve all observations for current time
obs_valid_now = []
for z in valid_times:
obs_valid_now.extend(fm10[z])
logging.info('FMDA found %d valid observations at model time %s' %
(len(obs_valid_now), str(model_time)))
fmc_gc = model.get_state()
dom_shape = fmc_gc.shape[:2]
# construct covariate storage
Xd3 = min(len(covariates) + 1, len(obs_valid_now))
logging.info('FMDA is using %d covariates' % Xd3)
X = np.zeros((dom_shape[0], dom_shape[1], Xd3))
X[:, :, 0] = fmc_gc[:, :, 1]
for i, c in zip(range(Xd3 - 1), covariates):
X[:, :, i + 1] = covariates[i]
# run the trend surface model (clamp output to [0.0 - 2.5] to be safe)
Kf_fn, Vf_fn = fit_tsm(obs_valid_now, X)
Kf_fn[Kf_fn < 0.0] = 0.0
Kf_fn[Kf_fn > 2.5] = 2.5
Kg = np.zeros((dom_shape[0], dom_shape[1], fmc_gc.shape[2]))
# run the data assimilation step now
logging.info(
"FMDA mean Kf: %g Vf: %g state[0]: %g state[1]: %g state[2]: %g" %
(np.mean(Kf_fn), np.mean(Vf_fn), np.mean(fmc_gc[:, :, 0]),
np.mean(fmc_gc[:, :, 1]), np.mean(fmc_gc[:, :, 2])))
model.kalman_update_single2(Kf_fn[:, :, np.newaxis],
Vf_fn[:, :, np.newaxis, np.newaxis], 1, Kg)
logging.info(
"FMDA mean Kf: %g Vf: %g state[0]: %g state[1]: %g state[2]: %g" %
(np.mean(Kf_fn), np.mean(Vf_fn), np.mean(fmc_gc[:, :, 0]),
np.mean(fmc_gc[:, :, 1]), np.mean(fmc_gc[:, :, 2])))
else:
logging.warning(
'FMDA no valid observations found, skipping data assimilation.')
def execute_da_step(model, model_time, covariates, fm10):
"""
Execute a single DA step from the current state/extended parameters and covariance matrix using
the <covariates> and observations <fm10>. Assimilation time window is fixed at 60 mins.
:param model: a FuelMoistureModel
:param model_time: the current model time
:param covariates: the covariate fields to take into account to model the spatial structure of the FM field
:param fm10: the 10-hr fuel moisture observations
"""
valid_times = [z for z in fm10.keys() if abs((z - model_time).total_seconds()) < 1800]
if len(valid_times) > 0:
# retrieve all observations for current time
obs_valid_now = []
for z in valid_times:
obs_valid_now.extend(fm10[z])
logging.info('FMDA found %d valid observations at model time %s' % (len(obs_valid_now), str(model_time)))
fmc_gc = model.get_state()
dom_shape = fmc_gc.shape[:2]
# construct covariate storage
Xd3 = min(len(covariates) + 1, len(obs_valid_now))
logging.info('FMDA is using %d covariates' % Xd3)
X = np.zeros((dom_shape[0], dom_shape[1], Xd3))
X[:,:,0] = fmc_gc[:,:,1]
for i,c in zip(range(Xd3-1),covariates):
X[:,:,i+1] = covariates[i]
# run the trend surface model (clamp output to [0.0 - 2.5] to be safe)
Kf_fn, Vf_fn = fit_tsm(obs_valid_now, X)
Kf_fn[Kf_fn < 0.0] = 0.0
Kf_fn[Kf_fn > 2.5] = 2.5
Kg = np.zeros((dom_shape[0], dom_shape[1], fmc_gc.shape[2]))
# run the data assimilation step now
logging.info("FMDA mean Kf: %g Vf: %g state[0]: %g state[1]: %g state[2]: %g" %
(np.mean(Kf_fn), np.mean(Vf_fn), np.mean(fmc_gc[:,:,0]), np.mean(fmc_gc[:,:,1]), np.mean(fmc_gc[:,:,2])))
model.kalman_update_single2(Kf_fn[:,:,np.newaxis], Vf_fn[:,:,np.newaxis,np.newaxis], 1, Kg)
logging.info("FMDA mean Kf: %g Vf: %g state[0]: %g state[1]: %g state[2]: %g" %
(np.mean(Kf_fn), np.mean(Vf_fn), np.mean(fmc_gc[:,:,0]), np.mean(fmc_gc[:,:,1]), np.mean(fmc_gc[:,:,2])))
else:
logging.warning('FMDA no valid observations found, skipping data assimilation.')
def execute_da_step(model, model_time, covariates, fm10):
"""
Execute a single DA step from the current state/extended parameters and covariance matrix using
the <covariates> and observations <fm10>. Assimilation time window is fixed at 60 mins.
"""
valid_times = [
z for z in fm10.keys() if abs(total_seconds(z - model_time)) < 1800
]
print('INFO: there are %d valid times at model time %s' %
(len(valid_times), str(model_time)))
if len(valid_times) > 0:
# retrieve all observations for current time
obs_valid_now = []
for z in valid_times:
obs_valid_now.extend(fm10[z])
for o in obs_valid_now:
print o
fmc_gc = model.get_state()
dom_shape = fmc_gc.shape[:2]
# construct covariate storage
Xd3 = len(covariates) + 1
X = np.zeros((dom_shape[0], dom_shape[1], Xd3))
X[:, :, 0] = fmc_gc[:, :, 1]
for c, i in zip(covariates, np.arange(len(covariates)) + 1):
X[:, :, i] = covariates[i - 1]
# run the trend surface model (clamp output to [0.0 - 2.5] to be safe)
Kf_fn, Vf_fn = fit_tsm(obs_valid_now, X)
Kf_fn[Kf_fn < 0.0] = 0.0
Kf_fn[Kf_fn > 2.5] = 2.5
# preallocate Kalman gain variable [not really used]
Kg = np.zeros((dom_shape[0], dom_shape[1], 5))
# run the data assimilation step now
print("Mean Kf: %g Vf: %g state[0]: %g state[1]: %g state[2]: %g\n" %
(np.mean(Kf_fn), np.mean(Vf_fn), np.mean(fmc_gc[:, :, 0]),
np.mean(fmc_gc[:, :, 1]), np.mean(fmc_gc[:, :, 2])))
model.kalman_update_single2(Kf_fn[:, :, np.newaxis],
Vf_fn[:, :, np.newaxis, np.newaxis], 1, Kg)
print("Mean Kf: %g Vf: %g state[0]: %g state[1]: %g state[2]: %g\n" %
(np.mean(Kf_fn), np.mean(Vf_fn), np.mean(fmc_gc[:, :, 0]),
np.mean(fmc_gc[:, :, 1]), np.mean(fmc_gc[:, :, 2])))
def execute_da_step(model, model_time, covariates, fm10):
"""
Execute a single DA step from the current state/extended parameters and covariance matrix using
the <covariates> and observations <fm10>. Assimilation time window is fixed at 60 mins.
"""
valid_times = [z for z in fm10.keys() if abs(total_seconds(z - model_time)) < 1800]
print('INFO: there are %d valid times at model time %s' % (len(valid_times), str(model_time)))
if len(valid_times) > 0:
# retrieve all observations for current time
obs_valid_now = []
for z in valid_times:
obs_valid_now.extend(fm10[z])
for o in obs_valid_now:
print o
fmc_gc = model.get_state()
dom_shape = fmc_gc.shape[:2]
# construct covariate storage
Xd3 = len(covariates) + 1
X = np.zeros((dom_shape[0], dom_shape[1], Xd3))
X[:,:,0] = fmc_gc[:,:,1]
for c,i in zip(covariates,np.arange(len(covariates))+1):
X[:,:,i] = covariates[i-1]
# run the trend surface model (clamp output to [0.0 - 2.5] to be safe)
Kf_fn, Vf_fn = fit_tsm(obs_valid_now, X)
Kf_fn[Kf_fn < 0.0] = 0.0
Kf_fn[Kf_fn > 2.5] = 2.5
# preallocate Kalman gain variable [not really used]
Kg = np.zeros((dom_shape[0], dom_shape[1], 5))
# run the data assimilation step now
print("Mean Kf: %g Vf: %g state[0]: %g state[1]: %g state[2]: %g\n" %
(np.mean(Kf_fn), np.mean(Vf_fn), np.mean(fmc_gc[:,:,0]), np.mean(fmc_gc[:,:,1]), np.mean(fmc_gc[:,:,2])))
model.kalman_update_single2(Kf_fn[:,:,np.newaxis], Vf_fn[:,:,np.newaxis,np.newaxis], 1, Kg)
print("Mean Kf: %g Vf: %g state[0]: %g state[1]: %g state[2]: %g\n" %
(np.mean(Kf_fn), np.mean(Vf_fn), np.mean(fmc_gc[:,:,0]), np.mean(fmc_gc[:,:,1]), np.mean(fmc_gc[:,:,2])))
def run_data_assimilation(in_dir0, in_dir1, fm_dir):
# load RTMA data for previous
print("INFO: loading RTMA data for time t-1 from [%s] ..." % in_dir0)
tm0 = time_from_dir(in_dir0)
tm = time_from_dir(in_dir1)
max_back = 6
data0 = None
while max_back > 0:
in_dir0 = 'inputs/%04d%02d%02d-%02d00' % (tm0.year, tm0.month, tm0.day, tm0.hour)
print('INFO: searching for RTMA in directory %s' % in_dir0)
data0 = load_rtma_data(in_dir0)
if data0 is not None:
break
print("WARN: cannot find RTMA data for time %s in directory %s, going back one hour" % (str(tm0),in_dir0))
max_back -= 1
tm0 = tm0 - timedelta(0,3600)
if data0 is None:
print("FATAL: cannot find a suitable previous RTMA analysis fror time %s." % str(tm))
return
print("INFO: loading RTMA data for time t from [%s] ..." % in_dir1)
data1 = load_rtma_data(in_dir1)
if data1 is None:
print('FATAL: insufficient environmnetal data for time %s, skipping ...' % tm)
return
# retrieve variables from RTMA
lat, lon, hgt = data0['Lat'], data0['Lon'], data0['HGT']
t20, relh0 = data0['T2'], data0['RH']
t21, relh1, rain = data1['T2'], data1['RH'], data1['RAIN']
ed0, ew0 = compute_equilibria(t20,relh0)
ed1, ew1 = compute_equilibria(t21,relh1)
tm0, tm = data0['Time'], data1['Time']
tm_str = tm.strftime('%Y%m%d-%H00')
tm_str0 = tm0.strftime('%Y%m%d-%H00')
# compute mean values for the Equilibria at t-1 and at t
ed = 0.5 * (ed0 + ed1)
ew = 0.5 * (ew0 + ew1)
dom_shape = lat.shape
print('INFO: domain size is %d x %d grid points.' % dom_shape)
print('INFO: domain extent is lats (%g to %g) lons (%g to %g).' % (np.amin(lat),np.amax(lat),np.amin(lon),np.amax(lon)))
print('INFO: stepping from time %s to time %s' % (tm0, tm))
# initialize output file
out_fm_file = os.path.join(fm_dir, 'fm-%s.nc' % tm_str)
out_file = netCDF4.Dataset(out_fm_file, 'w')
out_file.createDimension('fuel_moisture_classes_stag', 5)
out_file.createDimension('south_north', dom_shape[0])
out_file.createDimension('west_east', dom_shape[1])
nced = out_file.createVariable('Ed', 'f4', ('south_north', 'west_east'))
nced[:,:] = ed
ncew = out_file.createVariable('Ew', 'f4', ('south_north', 'west_east'))
ncew[:,:] = ew
ncfmc_fc = out_file.createVariable('FMC_GC_FC', 'f4', ('south_north', 'west_east','fuel_moisture_classes_stag'))
ncfmc_an = out_file.createVariable('FMC_GC', 'f4', ('south_north', 'west_east','fuel_moisture_classes_stag'))
nckg = out_file.createVariable('K', 'f4', ('south_north', 'west_east','fuel_moisture_classes_stag'))
ncfmc_cov = out_file.createVariable('FMC_COV', 'f4', ('south_north', 'west_east','fuel_moisture_classes_stag', 'fuel_moisture_classes_stag'))
ncrelh = out_file.createVariable('RELH','f4', ('south_north', 'west_east'))
ncrelh[:,:] = relh1
nctemp = out_file.createVariable('T2','f4', ('south_north', 'west_east'))
nctemp[:,:] = t21
nclat = out_file.createVariable('Lat', 'f4', ('south_north', 'west_east'))
nclat[:,:] = lat
nclon = out_file.createVariable('Lon', 'f4', ('south_north', 'west_east'))
nclon[:,:] = lon
print('INFO: opened %s and wrote XLAT,XLONG,RELH,T2,Ed,Ew fields.' % out_fm_file)
### Load observation data from the stations
# compute the diagonal distance between grid points
grid_dist_km = great_circle_distance(lon[0,0], lat[0,0], lon[1,1], lat[1,1])
print('INFO: diagonal distance in grid is %g' % grid_dist_km)
raws_path = os.path.join(in_dir1, 'raws_ingest_%4d%02d%02d-%02d%02d.csv' % (tm.year,tm.month,tm.day,tm.hour,tm.minute))
obss = load_raws_observations(raws_path,lat,lon)
print('INFO: Loaded %d observations.' % (len(obss)))
# set up parameters
Nk = 3 # we simulate 4 types of fuel
Q = np.diag([1e-4,5e-5,1e-5,1e-6,1e-6])
P0 = np.diag([0.01,0.01,0.01,0.001,0.001])
Tk = np.array([1.0, 10.0, 100.0])
dt = (tm - tm0).seconds
print("INFO: Time step is %d seconds." % dt)
# remove rain that is too small to make any difference
rain[rain < 0.01] = 0
# preprocess all covariates
X = np.zeros((dom_shape[0], dom_shape[1], 4))
X[:,:,1] = 1.0
X[:,:,2] = hgt / 2000.0
if np.any(rain) > 0.01:
X[:,:,3] = rain
else:
X = X[:,:,:3]
# load current state (or initialize from equilibrium if not found)
fm0 = None
fm_cov0 = None
in_fm_file = os.path.join(fm_dir, 'fm-%s.nc' % tm_str0)
if os.path.isfile(in_fm_file):
in_file = netCDF4.Dataset(in_fm_file, 'r')
fm0 = in_file.variables['FMC_GC'][:,:,:]
fm_cov0 = in_file.variables['FMC_COV'][:,:,:,:]
print('INFO: found input file %s, initializing from it [fm is %dx%dx%d, fm_cov is %dx%dx%dx%d]' %
(in_fm_file,fm0.shape[0],fm0.shape[1],fm0.shape[2],fm_cov0.shape[0],fm_cov0.shape[1],
fm_cov0.shape[2],fm_cov0.shape[3]))
in_file.close()
else:
print('INFO: input file %s not found, initializing from equilibrium' % in_fm_file)
fm0 = 0.5 * (ed + ew)
fm0 = fm0[:,:,np.newaxis][:,:,np.zeros((5,),dtype=np.int)]
fm0[:,:,3] = -0.04
fm0[:,:,4] = 0
fm_cov0 = P0
models = GridMoistureModel(fm0, Tk, 0.08, 2, 0.6, 7, fm_cov0)
print('INFO: performing forecast at: [time=%s].' % str(tm))
# compute the FORECAST
models.advance_model(ed, ew, rain, dt, Q)
f = models.get_state()
ncfmc_fc[:,:,:] = f
# fill 10-hr forecast as the first field of X
X[:,:,0] = f[:,:,1]
# examine the assimilated fields (if assimilation is activated)
for i in range(3):
print('INFO [%d]: [min %g, mean %g, max %g]' % (i, np.amin(f[:,:,i]), np.mean(f[:,:,i]), np.amax(f[:,:,i])))
if np.any(f[:,:,i] < 0.0):
print("WARN: in field %d there were %d negative moisture values !" % (i, np.count_nonzero(f[:,:,i] < 0.0)))
if np.any(f[:,:,i] > 0.6):
print("WARN: in field %d there were %d moisture values above 0.6!" % (i, np.count_nonzero(f[:,:,i] > 2.5)))
if len(obss) > 0:
print('INFO: running trend surface model ...')
# fit the trend surface model to data
tsm, tsm_var, s2 = fit_tsm(obss, X)
print('INFO: microscale variability variance is %g' % s2)
if np.count_nonzero(tsm > 0.6) > 0:
print('WARN: in TSM found %d values over 0.6, %d of those had rain, clamped to 2.5' %
(np.count_nonzero(tsm > 0.6),
np.count_nonzero(np.logical_and(tsm > 0.6, rain > 0.0))))
tsm[tsm > 0.6] = 0.6
if np.count_nonzero(tsm < 0.0) > 0:
print('WARN: in TSM found %d values under 0.0, clamped to 0.0' % np.count_nonzero(tsm < 0.0))
tsm[tsm < 0.0] = 0.0
print('INFO: running KF ...')
# run the kalman update step
Kg = np.zeros((dom_shape[0], dom_shape[1], len(Tk)+2))
models.kalman_update_single2(tsm[:,:,np.newaxis], tsm_var[:,:,np.newaxis,np.newaxis], 1, Kg)
# check post-assimilation results
f = models.get_state()
for i in range(3):
if np.any(f[:,:,i] < 0.0):
print("WARN: in field %d there were %d negative moisture values and %f values over 0.6 !" %
(i, np.count_nonzero(f[:,:,i] < 0.0),np.count_nonzero(f[:,:,i] > 0.6)))
f[f < 0] = 0
nckg[:,:,:] = Kg
print('INFO: storing results in netCDF file %s.' % out_fm_file)
# store post-assimilation (or forecast depending on whether observations were available) FM-10 state and variance
ncfmc_an[:,:,:] = f
ncfmc_cov[:,:,:,:] = models.get_state_covar()
# close the netCDF file (relevant if we did write into FMC_GC)
out_file.close()
print('INFO: SUCCESS')
def run_module():
# read in configuration file to execute run
print("Reading configuration from [%s]" % sys.argv[1])
with open(sys.argv[1]) as f:
cfg = eval(f.read())
# ensure output path exists
if not os.path.isdir(cfg['output_dir']):
os.mkdir(cfg['output_dir'])
# configure diagnostics
init_diagnostics(
os.path.join(cfg['output_dir'], 'moisture_model_v1_diagnostics.txt'))
# Trend surface model diagnostics
diagnostics().configure_tag("kriging_cov_cond", True, True, True)
diagnostics().configure_tag("s2_eta_hat", True, True, True)
diagnostics().configure_tag("kriging_rmse", True, True, True)
diagnostics().configure_tag("kriging_beta", True, True, True)
diagnostics().configure_tag("kriging_iters", False, True, True)
diagnostics().configure_tag("kriging_subzero_s2_estimates", False, True,
True)
diagnostics().configure_tag("fm10_kriging_var", True, True, True)
diagnostics().configure_tag("f0_summary", True, True, True)
diagnostics().configure_tag("f1_summary", True, True, True)
diagnostics().configure_tag("f2_summary", True, True, True)
diagnostics().configure_tag("f3_summary", True, True, True)
# Assimilation parameters
diagnostics().configure_tag("K0_summary", True, True, True)
diagnostics().configure_tag("K1_summary", True, True, True)
diagnostics().configure_tag("K2_summary", True, True, True)
diagnostics().configure_tag("K3_summary", True, True, True)
diagnostics().configure_tag("assim_info", False, False, True)
# Model forecast, analysis and non-assimilated model: state, covariance, errors
diagnostics().configure_tag("fm10f_rmse", True, True, True)
diagnostics().configure_tag("fm10na_rmse", True, True, True)
# all simulation times and all assimilation times (subset)
diagnostics().configure_tag("mta", False, True, True)
diagnostics().configure_tag("mt", False, True, True)
# observation values and their nearest grid points
diagnostics().configure_tag("obs_vals", False, True, True)
diagnostics().configure_tag("obs_ngp", False, True, True)
# in test mode, we will emit observations at the target station
# our predictions, the nearest grid point and the test station id
diagnostics().configure_tag("test_obs", True, True, True)
diagnostics().configure_tag("test_pred", True, True, True)
diagnostics().configure_tag("test_ngp", True, True, True)
diagnostics().configure_tag("test_station_id", True, True, True)
### Load and preprocess WRF model data
# load WRF data
wrf_data = WRFModelData(cfg['wrf_output'],
['T2', 'Q2', 'PSFC', 'RAINNC', 'RAINC', 'HGT'])
wrf_data.slice_field('HGT')
# read in spatial and temporal extent of WRF variables
lat, lon = wrf_data.get_lats(), wrf_data.get_lons()
hgt = wrf_data['HGT']
tm = wrf_data.get_gmt_times()
Nt = cfg['Nt'] if cfg.has_key('Nt') and cfg['Nt'] is not None else len(tm)
dom_shape = lat.shape
print('INFO: domain size is %d x %d grid points.' % dom_shape)
print('INFO: domain extent is lats (%g to %g) lons (%g to %g).' %
(np.amin(lat), np.amax(lat), np.amin(lon), np.amax(lon)))
# if writing is requested, open output file and set up dimensions
if cfg['write_fields'] not in ['all', 'fmc_gc', 'none']:
error('FATAL: write_fields must be one of all, fmc_gc or none.')
if cfg['write_fields'] == 'none':
cfg['write_fields'] = False
out_file = None
ncfmc_gc, ncfm10a, ncfm10aV, ncfm10f, cnfm10fV, ncfm10na = None, None, None, None, None, None
nctsmV, ncKg = None, None
if cfg['write_fields']:
out_file = netCDF4.Dataset(cfg['output_dir'] + '/fields.nc', 'w')
out_file.createDimension('Time', None)
out_file.createDimension('fuel_moisture_classes_stag', 5)
out_file.createDimension('south_north', dom_shape[0])
out_file.createDimension('west_east', dom_shape[1])
ncfmc_gc = out_file.createVariable(
'FMC_GC', 'f4',
('Time', 'fuel_moisture_classes_stag', 'south_north', 'west_east'))
if cfg['write_fields'] == 'all':
ncfm10a = out_file.createVariable(
'fm10a', 'f4', ('Time', 'south_north', 'west_east'))
ncfm10aV = out_file.createVariable(
'fm10a_var', 'f4', ('Time', 'south_north', 'west_east'))
ncfm10na = out_file.createVariable(
'fm10na', 'f4', ('Time', 'south_north', 'west_east'))
ncfm10f = out_file.createVariable(
'fm10f', 'f4', ('Time', 'south_north', 'west_east'))
ncfm10fV = out_file.createVariable(
'fm10f_var', 'f4', ('Time', 'south_north', 'west_east'))
nctsmV = out_file.createVariable(
'tsm_var', 'f4', ('Time', 'south_north', 'west_east'))
ncKg = out_file.createVariable(
'kalman_gain', 'f4', ('Time', 'south_north', 'west_east'))
print('INFO: opened fields.nc for writing ALL output fields.')
else:
print('INFO: opened field.nc for writing FMC_GC only.')
test_mode = (cfg['run_mode'] == 'test')
tgt_station = None
if cfg['run_mode'] == 'test':
print('INFO: running in TEST mode! Will perform leave-one-out tesing.')
tgt_station_id = cfg['target_station_id']
diagnostics().push('test_station_id', tgt_station_id)
elif cfg['run_mode'] == 'production':
print(
'INFO: running in PRODUCTION mode! Using all observation stations.'
)
else:
error('FATAL: invalid run mode! Must be "test" or "production".')
# determine simulation times
tm_start = parse_datetime(
cfg['start_time']) if cfg['start_time'] is not None else tm[0]
tm_end = parse_datetime(
cfg['end_time']) if cfg['end_time'] is not None else tm[-1]
# if the required start time or end time are outside the simulation domain, exit with an error
if tm_start < tm[0] or tm_end > tm[-1]:
print('FATAL: invalid time range, required [%s-%s], availble [%s-%s]' %
(str(tm_start), str(tm_end), str(tm[0]), str(tm[-1])))
sys.exit(2)
print('INFO: time limits are %s to %s\nINFO: simulation is from %s to %s' %
(str(tm_start), str(tm_end), str(tm[0]), str(tm[-1])))
# retrieve dynamic covariates and remove mean at each time point for T2 and PSFC
T2 = wrf_data['T2']
#T2 -= np.mean(np.mean(T2,axis=0),axis=0)[np.newaxis,np.newaxis,:]
PSFC = wrf_data['PSFC']
#PSFC -= np.mean(np.mean(PSFC,axis=0),axis=0)[np.newaxis,np.newaxis,:]
# numerical fix - if it rains at an intensity of less than 0.001 per hour, set rain to zero
# also, use log(rain + 1) to prevent wild trend surface model predictions when stations see little rain
# but elsewhere there is too much rain
# without this, numerical errors in trend surface model may pop up
rain = wrf_data['RAIN']
#rain[rain < 0.01] = 0.0
rain = np.log(rain + 1.0)
# moisture equilibria are now computed from averaged Q,P,T at beginning and end of period
Ed, Ew = wrf_data.get_moisture_equilibria()
### Load observation data from the stations
# compute the diagonal distance between grid points
grid_dist_km = great_circle_distance(lon[0, 0], lat[0, 0], lon[1, 1],
lat[1, 1])
print('INFO: diagonal distance in grid is %g' % grid_dist_km)
# load station data from files
with open(cfg['station_list_file'], 'r') as f:
si_list = f.read().split('\n')
si_list = filter(lambda x: len(x) > 0 and x[0] != '#',
map(string.strip, si_list))
# for each station id, load the station
stations = []
for code in si_list:
mws = MesoWestStation(code)
mws.load_station_info(
os.path.join(cfg["station_info_dir"], "%s.info" % code))
mws.register_to_grid(wrf_data)
if mws.get_dist_to_grid() < grid_dist_km / 2.0:
print(
'Station %s: lat %g lon %g nearest grid pt %s lat %g lon %g dist_to_grid %g'
% (code, mws.lat, mws.lon, str(mws.grid_pt), lat[mws.grid_pt],
lon[mws.grid_pt], mws.dist_grid_pt))
mws.load_station_data(
os.path.join(cfg["station_data_dir"], "%s.obs" % code))
if test_mode and mws.get_id() == tgt_station_id:
tgt_station = mws
print(
'INFO: in test mode, targeting station %s (removed from data pool).'
% tgt_station_id)
diagnostics().push("test_ngp", mws.get_nearest_grid_point())
else:
stations.append(mws)
print('Loaded %d stations (discarded %d stations, too far from grid).' %
(len(stations), len(si_list) - len(stations)))
if test_mode and tgt_station is None:
error(
'FATAL: in test mode, a station was removed that was not among accepted stations.'
)
# build the observation data
obs_data_fm10 = build_observation_data(stations, 'FM')
# build target data if in test mode
tgt_obs_fm10 = None
test_ngp = None
if test_mode:
test_ngp = tgt_station.get_nearest_grid_point()
tgt_obs_fm10 = build_observation_data([tgt_station], 'FM')
### Initialize model and visualization
# construct initial conditions from timestep 0
E = 0.5 * (Ed[0, :, :] + Ew[0, :, :])
# set up parameters
Nk = 4 # we simulate 4 types of fuel
Q = np.diag(cfg['Q'])
P0 = np.diag(cfg['P0'])
Tk = np.array([1.0, 10.0, 100.0, 1000.0]) * 3600
dt = (tm[1] - tm[0]).seconds
print("INFO: Computed timestep from WRF is is %g seconds." % dt)
mresV = np.zeros_like(E)
mid = np.zeros_like(E)
Kg = np.zeros((dom_shape[0], dom_shape[1], len(Tk) + 2))
# preprocess all static covariates
cov_ids = cfg['covariates']
Xd3 = len(cov_ids) + 1
X = np.zeros((dom_shape[0], dom_shape[1], Xd3))
Xr = np.zeros((dom_shape[0], dom_shape[1], Xd3))
static_covar_map = {
"lon": lon - np.mean(lon),
"lat": lat - np.mean(lat),
"elevation": hgt - np.mean(hgt),
"constant": np.ones(dom_shape)
}
dynamic_covar_map = {"temperature": T2, "pressure": PSFC, "rain": rain}
for i in range(1, Xd3):
cov_id = cov_ids[i - 1]
if cov_id in static_covar_map:
print('INFO: found static covariate %s' % cov_id)
Xr[:, :, i] = static_covar_map[cov_id]
elif cov_id in dynamic_covar_map:
print('INFO: found dynamic covariate %s' % cov_id)
else:
print('FATAL: unknown covariate %s encountered' % cov_id)
sys.exit(2)
print("INFO: there are %d covariates (including model state)" % Xd3)
# retrieve assimilation time window
assim_time_win = cfg['assimilation_time_window']
print('GMM init: equilibrium (%g,%g,%g) and at 86,205 %g' %
(np.amin(E), np.mean(E), np.amax(E), E[86, 205]))
models = GridMoistureModel(
E[:, :, np.newaxis][:, :, np.zeros((4, ), dtype=np.int)], Tk, P0)
models_na = GridMoistureModel(
E[:, :, np.newaxis][:, :, np.zeros((4, ), dtype=np.int)], Tk, P0)
### Run model for each WRF timestep and assimilate data when available
t_start, t_end = 1, len(tm) - 1
while tm_start > tm[t_start]:
t_start += 1
while tm_end < tm[t_end]:
t_end -= 1
# the first FMC_GC value gets filled out with equilibria
if cfg['write_fields']:
for i in range(Nk):
ncfmc_gc[0, i, :, :] = E
print('INFO: running simulation from %s (%d) to %s (%d).' %
(str(tm[t_start]), t_start, str(tm[t_end]), t_end))
for t in range(t_start, t_end + 1):
model_time = tm[t]
print("INFO: time: %s, step: %d" % (str(model_time), t))
diagnostics().push("mt", model_time)
models_na.advance_model(Ed[t - 1, :, :], Ew[t - 1, :, :],
rain[t - 1, :, :], dt, Q)
models.advance_model(Ed[t - 1, :, :], Ew[t - 1, :, :],
rain[t - 1, :, :], dt, Q)
# extract fuel moisture contents [make a fresh copy every iteration!]
f = models.get_state().copy()
f_na = models_na.get_state().copy()
# push 10-hr fuel state & variance of forecast
if cfg['write_fields'] == 'all':
ncfm10f[t, :, :] = models.get_state()[:, :, 1]
ncfm10fV[t, :, :] = models.P[:, :, 1, 1]
ncfm10na[t, :, :] = models_na.get_state()[:, :, 1]
# examine the assimilated fields (if assimilation is activated)
for i in range(4):
diagnostics().push("f%d_summary" % i, (t, np.amin(
f[:, :, i]), np.mean(f[:, :, i]), np.amax(f[:, :, i])))
if np.any(f[:, :, i] < 0.0):
print(
"WARN: in field %d there were %d negative moisture values !"
% (i, np.count_nonzero(f[:, :, i] < 0.0)))
ind = np.unravel_index(np.argmin(f[:, :, i]), f.shape[:2])
print(models.P[ind[0], ind[1], :, :])
print("Full model state at position %d,%d:" % (ind[0], ind[1]))
print(models.m_ext[ind[0], ind[1], :])
if np.any(f[:, :, i] > 2.5):
print(
"WARN: in field %d there were %d moisture values above 2.5!"
% (i, np.count_nonzero(f[:, :, i] > 2.5)))
ind = np.unravel_index(np.argmax(f[:, :, i]), f.shape[:2])
print(models.P[ind[0], ind[1], :, :])
print("Full model state at position %d,%d:" % (ind[0], ind[1]))
print(models.m_ext[ind[0], ind[1], :])
if cfg['assimilate']:
# run Kriging on each observed fuel type
Kfs, Vfs, fns = [], [], []
for obs_data, fuel_ndx in [(obs_data_fm10, 1)]:
# run the kriging subsystem and the Kalman update only if have valid observations
valid_times = [
z for z in obs_data.keys()
if abs(total_seconds(z - model_time)) < assim_time_win /
2.0
]
print(
'INFO: there are %d valid times at model time %s for fuel index %d'
% (len(valid_times), str(model_time), fuel_ndx))
if len(valid_times) > 0:
# add model time as time when assimilation occurred
diagnostics().push("mta", model_time)
# retrieve observations for current time
obs_valid_now = []
for z in valid_times:
obs_valid_now.extend(obs_data[z])
print(
'INFO: model time %s, assimilating %d observations.' %
(str(model_time), len(obs_valid_now)))
# construct covariates for this time instant
X[:, :, 0] = f[:, :, fuel_ndx]
for i in range(1, Xd3):
cov_id = cov_ids[i - 1]
if cov_id in static_covar_map:
X[:, :, i] = Xr[:, :, i]
elif cov_id in dynamic_covar_map:
F = dynamic_covar_map[cov_id]
X[:, :, i] = F[t, :, :]
else:
error('FATAL: found unknown covariate %s' % cov_id)
# find differences (residuals) between observed measurements and nearest grid points
obs_vals = [o.get_value() for o in obs_valid_now]
obs_ngp = [
o.get_nearest_grid_point() for o in obs_valid_now
]
diagnostics().push("obs_vals", obs_vals)
diagnostics().push("obs_ngp", obs_ngp)
mod_vals = np.array(
[f[i, j, fuel_ndx] for i, j in obs_ngp])
mod_na_vals = np.array(
[f_na[i, j, fuel_ndx] for i, j in obs_ngp])
diagnostics().push("fm10f_rmse",
np.mean((obs_vals - mod_vals)**2)**0.5)
diagnostics().push(
"fm10na_rmse",
np.mean((obs_vals - mod_na_vals)**2)**0.5)
# krige observations to grid points
Kf_fn, Vf_fn = fit_tsm(obs_valid_now, X)
if np.count_nonzero(Kf_fn > 2.5) > 0:
rain_t = dynamic_covar_map['rain'][t, :, :]
print(
'WARN: in TSM found %d values over 2.5, %d of those had rain, clamped to 2.5'
% (np.count_nonzero(Kf_fn > 2.5),
np.count_nonzero(
np.logical_and(Kf_fn > 2.5, rain_t > 0.0))))
Kf_fn[Kf_fn > 2.5] = 2.5
if np.count_nonzero(Kf_fn < 0.0) > 0:
print(
'WARN: in TSM found %d values under 0.0, clamped to 0.0'
% np.count_nonzero(Kf_fn < 0.0))
Kf_fn[Kf_fn < 0.0] = 0.0
krig_vals = np.array([Kf_fn[ngp] for ngp in obs_ngp])
diagnostics().push("assim_info",
(t, fuel_ndx, obs_vals, krig_vals,
mod_vals, mod_na_vals))
diagnostics().push("fm10_kriging_var", (t, np.mean(Vf_fn)))
if cfg['write_fields'] == 'all':
nctsmV[t, :, :] = Vf_fn
# append to storage for kriged fields in this time instant
Kfs.append(Kf_fn)
Vfs.append(Vf_fn)
fns.append(fuel_ndx)
# if there were any observations, run the kalman update step
if len(fns) > 0:
NobsClasses = len(fns)
O = np.zeros((dom_shape[0], dom_shape[1], NobsClasses))
V = np.zeros(
(dom_shape[0], dom_shape[1], NobsClasses, NobsClasses))
for i in range(NobsClasses):
O[:, :, i] = Kfs[i]
V[:, :, i, i] = Vfs[i]
# execute the Kalman update
if len(fns) == 1:
models.kalman_update_single2(O, V, fns[0], Kg)
else:
models.kalman_update(O, V, fns, Kg)
# push new diagnostic outputs
if cfg['write_fields'] == 'all':
ncKg[t, :, :] = Kg[:, :, 1]
for i in range(4):
diagnostics().push(
"K%d_summary" % i,
(t, np.amin(Kg[:, :, i]), np.mean(
Kg[:, :, i]), np.amax(Kg[:, :, i])))
if np.any(models.get_state()[:, :, i] < 0.0):
print(
"WARN: in field %d there were %d negative moisture values !"
% (i,
np.count_nonzero(
models.get_state()[:, :, i] < 0.0)))
ind = np.unravel_index(
np.argmin(models.get_state()[:, :, i]),
models.get_state().shape[:2])
print(models.P[ind[0], ind[1], :, :])
print(
"TSM input at given position: value %g variance %g"
% (O[ind[0], ind[1]], V[ind[0], ind[1]]))
print("Model state at given position:")
print(models.m_ext[ind[0], ind[1], :])
# store post-assimilation (or forecast depending on whether observations were available) FM-10 state and variance
if cfg['write_fields'] == 'all':
ncfm10a[t, :, :] = models.get_state()[:, :, 1]
ncfm10aV[t, :, :] = models.P[:, :, 1, 1]
# we don't care if we assimilated or not, we always check our error on target station if in test mode
if test_mode:
valid_times = [
z for z in tgt_obs_fm10.keys()
if abs(total_seconds(z - model_time)) < assim_time_win /
2.0
]
tgt_i, tgt_j = test_ngp
diagnostics().push("test_pred", f[tgt_i, tgt_j, 1])
if len(valid_times) > 0:
# this is our target observation [FIXME: this disregards multiple observations if multiple happen to be valid]
tgt_obs = tgt_obs_fm10[valid_times[0]][0]
obs = tgt_obs.get_value()
diagnostics().push("test_obs", obs)
else:
diagnostics().push("test_obs", np.nan)
# store data in wrf_file variable FMC_G
if cfg['write_fields']:
ncfmc_gc[t, :Nk, :, :] = np.transpose(
models.get_state()[:, :, :Nk], axes=[2, 0, 1])
# store the diagnostics in a binary file when done
diagnostics().dump_store(os.path.join(cfg['output_dir'],
'diagnostics.bin'))
# close the netCDF file (relevant if we did write into FMC_GC)
if out_file is not None:
out_file.close()
def run_module():
# read in configuration file to execute run
print("Reading configuration from [%s]" % sys.argv[1])
with open(sys.argv[1]) as f:
cfg = eval(f.read())
# ensure output path exists
if not os.path.isdir(cfg['output_dir']):
os.mkdir(cfg['output_dir'])
# configure diagnostics
init_diagnostics(os.path.join(cfg['output_dir'], 'moisture_model_v1_diagnostics.txt'))
# Trend surface model diagnostics
diagnostics().configure_tag("kriging_cov_cond", True, True, True)
diagnostics().configure_tag("s2_eta_hat", True, True, True)
diagnostics().configure_tag("kriging_rmse", True, True, True)
diagnostics().configure_tag("kriging_beta", True, True, True)
diagnostics().configure_tag("kriging_iters", False, True, True)
diagnostics().configure_tag("kriging_subzero_s2_estimates", False, True, True)
diagnostics().configure_tag("fm10_kriging_var", True, True, True)
diagnostics().configure_tag("f0_summary", True, True, True)
diagnostics().configure_tag("f1_summary", True, True, True)
diagnostics().configure_tag("f2_summary", True, True, True)
diagnostics().configure_tag("f3_summary", True, True, True)
# Assimilation parameters
diagnostics().configure_tag("K0_summary", True, True, True)
diagnostics().configure_tag("K1_summary", True, True, True)
diagnostics().configure_tag("K2_summary", True, True, True)
diagnostics().configure_tag("K3_summary", True, True, True)
diagnostics().configure_tag("assim_info", False, False, True)
# Model forecast, analysis and non-assimilated model: state, covariance, errors
diagnostics().configure_tag("fm10f_rmse", True, True, True)
diagnostics().configure_tag("fm10na_rmse", True, True, True)
# all simulation times and all assimilation times (subset)
diagnostics().configure_tag("mta", False, True, True)
diagnostics().configure_tag("mt", False, True, True)
# observation values and their nearest grid points
diagnostics().configure_tag("obs_vals", False, True, True)
diagnostics().configure_tag("obs_ngp", False, True, True)
# in test mode, we will emit observations at the target station
# our predictions, the nearest grid point and the test station id
diagnostics().configure_tag("test_obs", True, True, True)
diagnostics().configure_tag("test_pred", True, True, True)
diagnostics().configure_tag("test_ngp", True, True, True)
diagnostics().configure_tag("test_station_id", True, True, True)
### Load and preprocess WRF model data
# load WRF data
wrf_data = WRFModelData(cfg['wrf_output'], ['T2', 'Q2', 'PSFC', 'RAINNC', 'RAINC', 'HGT'])
wrf_data.slice_field('HGT')
# read in spatial and temporal extent of WRF variables
lat, lon = wrf_data.get_lats(), wrf_data.get_lons()
hgt = wrf_data['HGT']
tm = wrf_data.get_gmt_times()
Nt = cfg['Nt'] if cfg.has_key('Nt') and cfg['Nt'] is not None else len(tm)
dom_shape = lat.shape
print('INFO: domain size is %d x %d grid points.' % dom_shape)
print('INFO: domain extent is lats (%g to %g) lons (%g to %g).' % (np.amin(lat),np.amax(lat),np.amin(lon),np.amax(lon)))
# if writing is requested, open output file and set up dimensions
if cfg['write_fields'] not in [ 'all', 'fmc_gc', 'none']:
error('FATAL: write_fields must be one of all, fmc_gc or none.')
if cfg['write_fields'] == 'none':
cfg['write_fields'] = False
out_file = None
ncfmc_gc, ncfm10a, ncfm10aV, ncfm10f, cnfm10fV, ncfm10na = None, None, None, None, None, None
nctsmV, ncKg = None, None
if cfg['write_fields']:
out_file = netCDF4.Dataset(cfg['output_dir'] + '/fields.nc', 'w')
out_file.createDimension('Time', None)
out_file.createDimension('fuel_moisture_classes_stag', 5)
out_file.createDimension('south_north', dom_shape[0])
out_file.createDimension('west_east', dom_shape[1])
ncfmc_gc = out_file.createVariable('FMC_GC', 'f4', ('Time', 'fuel_moisture_classes_stag', 'south_north', 'west_east'))
if cfg['write_fields'] == 'all':
ncfm10a = out_file.createVariable('fm10a', 'f4', ('Time', 'south_north', 'west_east'))
ncfm10aV = out_file.createVariable('fm10a_var', 'f4', ('Time', 'south_north', 'west_east'))
ncfm10na = out_file.createVariable('fm10na', 'f4', ('Time', 'south_north', 'west_east'))
ncfm10f = out_file.createVariable('fm10f', 'f4', ('Time', 'south_north', 'west_east'))
ncfm10fV = out_file.createVariable('fm10f_var', 'f4', ('Time', 'south_north', 'west_east'))
nctsmV = out_file.createVariable('tsm_var', 'f4', ('Time', 'south_north', 'west_east'))
ncKg = out_file.createVariable('kalman_gain', 'f4', ('Time', 'south_north', 'west_east'))
print('INFO: opened fields.nc for writing ALL output fields.')
else:
print('INFO: opened field.nc for writing FMC_GC only.')
test_mode = (cfg['run_mode'] == 'test')
tgt_station = None
if cfg['run_mode'] == 'test':
print('INFO: running in TEST mode! Will perform leave-one-out tesing.')
tgt_station_id = cfg['target_station_id']
diagnostics().push('test_station_id', tgt_station_id)
elif cfg['run_mode'] == 'production':
print('INFO: running in PRODUCTION mode! Using all observation stations.')
else:
error('FATAL: invalid run mode! Must be "test" or "production".')
# determine simulation times
tm_start = parse_datetime(cfg['start_time']) if cfg['start_time'] is not None else tm[0]
tm_end = parse_datetime(cfg['end_time']) if cfg['end_time'] is not None else tm[-1]
# if the required start time or end time are outside the simulation domain, exit with an error
if tm_start < tm[0] or tm_end > tm[-1]:
print('FATAL: invalid time range, required [%s-%s], availble [%s-%s]' %
(str(tm_start), str(tm_end), str(tm[0]), str(tm[-1])))
sys.exit(2)
print('INFO: time limits are %s to %s\nINFO: simulation is from %s to %s' %
(str(tm_start), str(tm_end), str(tm[0]), str(tm[-1])))
# retrieve dynamic covariates and remove mean at each time point for T2 and PSFC
T2 = wrf_data['T2']
#T2 -= np.mean(np.mean(T2,axis=0),axis=0)[np.newaxis,np.newaxis,:]
PSFC = wrf_data['PSFC']
#PSFC -= np.mean(np.mean(PSFC,axis=0),axis=0)[np.newaxis,np.newaxis,:]
# numerical fix - if it rains at an intensity of less than 0.001 per hour, set rain to zero
# also, use log(rain + 1) to prevent wild trend surface model predictions when stations see little rain
# but elsewhere there is too much rain
# without this, numerical errors in trend surface model may pop up
rain = wrf_data['RAIN']
#rain[rain < 0.01] = 0.0
rain = np.log(rain + 1.0)
# moisture equilibria are now computed from averaged Q,P,T at beginning and end of period
Ed, Ew = wrf_data.get_moisture_equilibria()
### Load observation data from the stations
# compute the diagonal distance between grid points
grid_dist_km = great_circle_distance(lon[0,0], lat[0,0], lon[1,1], lat[1,1])
print('INFO: diagonal distance in grid is %g' % grid_dist_km)
# load station data from files
with open(cfg['station_list_file'], 'r') as f:
si_list = f.read().split('\n')
si_list = filter(lambda x: len(x) > 0 and x[0] != '#', map(string.strip, si_list))
# for each station id, load the station
stations = []
for code in si_list:
mws = MesoWestStation(code)
mws.load_station_info(os.path.join(cfg["station_info_dir"], "%s.info" % code))
mws.register_to_grid(wrf_data)
if mws.get_dist_to_grid() < grid_dist_km / 2.0:
print('Station %s: lat %g lon %g nearest grid pt %s lat %g lon %g dist_to_grid %g' %
(code, mws.lat, mws.lon, str(mws.grid_pt), lat[mws.grid_pt], lon[mws.grid_pt], mws.dist_grid_pt))
mws.load_station_data(os.path.join(cfg["station_data_dir"], "%s.obs" % code))
if test_mode and mws.get_id() == tgt_station_id:
tgt_station = mws
print('INFO: in test mode, targeting station %s (removed from data pool).' % tgt_station_id)
diagnostics().push("test_ngp", mws.get_nearest_grid_point())
else:
stations.append(mws)
print('Loaded %d stations (discarded %d stations, too far from grid).' % (len(stations), len(si_list) - len(stations)))
if test_mode and tgt_station is None:
error('FATAL: in test mode, a station was removed that was not among accepted stations.')
# build the observation data
obs_data_fm10 = build_observation_data(stations, 'FM')
# build target data if in test mode
tgt_obs_fm10 = None
test_ngp = None
if test_mode:
test_ngp = tgt_station.get_nearest_grid_point()
tgt_obs_fm10 = build_observation_data([tgt_station], 'FM')
### Initialize model and visualization
# construct initial conditions from timestep 0
E = 0.5 * (Ed[0,:,:] + Ew[0,:,:])
# set up parameters
Nk = 4 # we simulate 4 types of fuel
Q = np.diag(cfg['Q'])
P0 = np.diag(cfg['P0'])
Tk = np.array([1.0, 10.0, 100.0, 1000.0]) * 3600
dt = (tm[1] - tm[0]).seconds
print("INFO: Computed timestep from WRF is is %g seconds." % dt)
mresV = np.zeros_like(E)
mid = np.zeros_like(E)
Kg = np.zeros((dom_shape[0], dom_shape[1], len(Tk)+2))
# preprocess all static covariates
cov_ids = cfg['covariates']
Xd3 = len(cov_ids) + 1
X = np.zeros((dom_shape[0], dom_shape[1], Xd3))
Xr = np.zeros((dom_shape[0], dom_shape[1], Xd3))
static_covar_map = { "lon" : lon - np.mean(lon), "lat" : lat - np.mean(lat), "elevation" : hgt - np.mean(hgt), "constant" : np.ones(dom_shape) }
dynamic_covar_map = { "temperature" : T2, "pressure" : PSFC, "rain" : rain }
for i in range(1, Xd3):
cov_id = cov_ids[i-1]
if cov_id in static_covar_map:
print('INFO: found static covariate %s' % cov_id)
Xr[:, :, i] = static_covar_map[cov_id]
elif cov_id in dynamic_covar_map:
print('INFO: found dynamic covariate %s' % cov_id)
else:
print('FATAL: unknown covariate %s encountered' % cov_id)
sys.exit(2)
print("INFO: there are %d covariates (including model state)" % Xd3)
# retrieve assimilation time window
assim_time_win = cfg['assimilation_time_window']
print('GMM init: equilibrium (%g,%g,%g) and at 86,205 %g' % (np.amin(E),np.mean(E),np.amax(E),E[86,205]))
models = GridMoistureModel(E[:,:,np.newaxis][:,:,np.zeros((4,),dtype=np.int)], Tk, P0)
models_na = GridMoistureModel(E[:,:,np.newaxis][:,:,np.zeros((4,),dtype=np.int)], Tk, P0)
### Run model for each WRF timestep and assimilate data when available
t_start, t_end = 1, len(tm)-1
while tm_start > tm[t_start]:
t_start+=1
while tm_end < tm[t_end]:
t_end-=1
# the first FMC_GC value gets filled out with equilibria
if cfg['write_fields']:
for i in range(Nk):
ncfmc_gc[0, i, :, :] = E
print('INFO: running simulation from %s (%d) to %s (%d).' % (str(tm[t_start]), t_start, str(tm[t_end]), t_end))
for t in range(t_start, t_end+1):
model_time = tm[t]
print("INFO: time: %s, step: %d" % (str(model_time), t))
diagnostics().push("mt", model_time)
models_na.advance_model(Ed[t-1,:,:], Ew[t-1,:,:], rain[t-1,:,:], dt, Q)
models.advance_model(Ed[t-1,:,:], Ew[t-1,:,:], rain[t-1,:,:], dt, Q)
# extract fuel moisture contents [make a fresh copy every iteration!]
f = models.get_state().copy()
f_na = models_na.get_state().copy()
# push 10-hr fuel state & variance of forecast
if cfg['write_fields'] == 'all':
ncfm10f[t,:,:] = models.get_state()[:,:,1]
ncfm10fV[t,:,:] = models.P[:,:,1,1]
ncfm10na[t,:,:] = models_na.get_state()[:,:,1]
# examine the assimilated fields (if assimilation is activated)
for i in range(4):
diagnostics().push("f%d_summary" % i, (t, np.amin(f[:,:,i]), np.mean(f[:,:,i]), np.amax(f[:,:,i])))
if np.any(f[:,:,i] < 0.0):
print("WARN: in field %d there were %d negative moisture values !" % (i, np.count_nonzero(f[:,:,i] < 0.0)))
ind = np.unravel_index(np.argmin(f[:,:,i]), f.shape[:2])
print(models.P[ind[0],ind[1],:,:])
print("Full model state at position %d,%d:" % (ind[0],ind[1]))
print(models.m_ext[ind[0],ind[1],:])
if np.any(f[:,:,i] > 2.5):
print("WARN: in field %d there were %d moisture values above 2.5!" % (i, np.count_nonzero(f[:,:,i] > 2.5)))
ind = np.unravel_index(np.argmax(f[:,:,i]), f.shape[:2])
print(models.P[ind[0],ind[1],:,:])
print("Full model state at position %d,%d:" % (ind[0],ind[1]))
print(models.m_ext[ind[0],ind[1],:])
if cfg['assimilate']:
# run Kriging on each observed fuel type
Kfs, Vfs, fns = [], [], []
for obs_data, fuel_ndx in [ (obs_data_fm10, 1) ]:
# run the kriging subsystem and the Kalman update only if have valid observations
valid_times = [z for z in obs_data.keys() if abs(total_seconds(z - model_time)) < assim_time_win/2.0]
print('INFO: there are %d valid times at model time %s for fuel index %d' % (len(valid_times), str(model_time), fuel_ndx))
if len(valid_times) > 0:
# add model time as time when assimilation occurred
diagnostics().push("mta", model_time)
# retrieve observations for current time
obs_valid_now = []
for z in valid_times:
obs_valid_now.extend(obs_data[z])
print('INFO: model time %s, assimilating %d observations.' % (str(model_time), len(obs_valid_now)))
# construct covariates for this time instant
X[:,:,0] = f[:,:,fuel_ndx]
for i in range(1, Xd3):
cov_id = cov_ids[i-1]
if cov_id in static_covar_map:
X[:, :, i] = Xr[:, :, i]
elif cov_id in dynamic_covar_map:
F = dynamic_covar_map[cov_id]
X[:, :, i] = F[t, :, :]
else:
error('FATAL: found unknown covariate %s' % cov_id)
# find differences (residuals) between observed measurements and nearest grid points
obs_vals = [o.get_value() for o in obs_valid_now]
obs_ngp = [o.get_nearest_grid_point() for o in obs_valid_now]
diagnostics().push("obs_vals", obs_vals)
diagnostics().push("obs_ngp", obs_ngp)
mod_vals = np.array([f[i,j,fuel_ndx] for i,j in obs_ngp])
mod_na_vals = np.array([f_na[i,j,fuel_ndx] for i,j in obs_ngp])
diagnostics().push("fm10f_rmse", np.mean((obs_vals - mod_vals)**2)**0.5)
diagnostics().push("fm10na_rmse", np.mean((obs_vals - mod_na_vals)**2)**0.5)
# krige observations to grid points
Kf_fn, Vf_fn = fit_tsm(obs_valid_now, X)
if np.count_nonzero(Kf_fn > 2.5) > 0:
rain_t = dynamic_covar_map['rain'][t,:,:]
print('WARN: in TSM found %d values over 2.5, %d of those had rain, clamped to 2.5' %
(np.count_nonzero(Kf_fn > 2.5),
np.count_nonzero(np.logical_and(Kf_fn > 2.5, rain_t > 0.0))))
Kf_fn[Kf_fn > 2.5] = 2.5
if np.count_nonzero(Kf_fn < 0.0) > 0:
print('WARN: in TSM found %d values under 0.0, clamped to 0.0' % np.count_nonzero(Kf_fn < 0.0))
Kf_fn[Kf_fn < 0.0] = 0.0
krig_vals = np.array([Kf_fn[ngp] for ngp in obs_ngp])
diagnostics().push("assim_info", (t, fuel_ndx, obs_vals, krig_vals, mod_vals, mod_na_vals))
diagnostics().push("fm10_kriging_var", (t, np.mean(Vf_fn)))
if cfg['write_fields'] == 'all':
nctsmV[t,:,:] = Vf_fn
# append to storage for kriged fields in this time instant
Kfs.append(Kf_fn)
Vfs.append(Vf_fn)
fns.append(fuel_ndx)
# if there were any observations, run the kalman update step
if len(fns) > 0:
NobsClasses = len(fns)
O = np.zeros((dom_shape[0], dom_shape[1], NobsClasses))
V = np.zeros((dom_shape[0], dom_shape[1], NobsClasses, NobsClasses))
for i in range(NobsClasses):
O[:,:,i] = Kfs[i]
V[:,:,i,i] = Vfs[i]
# execute the Kalman update
if len(fns) == 1:
models.kalman_update_single2(O, V, fns[0], Kg)
else:
models.kalman_update(O, V, fns, Kg)
# push new diagnostic outputs
if cfg['write_fields'] == 'all':
ncKg[t,:,:] = Kg[:,:,1]
for i in range(4):
diagnostics().push("K%d_summary" % i, (t, np.amin(Kg[:,:,i]), np.mean(Kg[:,:,i]), np.amax(Kg[:,:,i])))
if np.any(models.get_state()[:,:,i] < 0.0):
print("WARN: in field %d there were %d negative moisture values !" % (i, np.count_nonzero(models.get_state()[:,:,i] < 0.0)))
ind = np.unravel_index(np.argmin(models.get_state()[:,:,i]), models.get_state().shape[:2])
print(models.P[ind[0],ind[1],:,:])
print("TSM input at given position: value %g variance %g" % (O[ind[0],ind[1]], V[ind[0],ind[1]]))
print("Model state at given position:")
print(models.m_ext[ind[0],ind[1],:])
# store post-assimilation (or forecast depending on whether observations were available) FM-10 state and variance
if cfg['write_fields'] == 'all':
ncfm10a[t,:,:] = models.get_state()[:,:,1]
ncfm10aV[t,:,:] = models.P[:,:,1,1]
# we don't care if we assimilated or not, we always check our error on target station if in test mode
if test_mode:
valid_times = [z for z in tgt_obs_fm10.keys() if abs(total_seconds(z - model_time)) < assim_time_win/2.0]
tgt_i, tgt_j = test_ngp
diagnostics().push("test_pred", f[tgt_i, tgt_j,1])
if len(valid_times) > 0:
# this is our target observation [FIXME: this disregards multiple observations if multiple happen to be valid]
tgt_obs = tgt_obs_fm10[valid_times[0]][0]
obs = tgt_obs.get_value()
diagnostics().push("test_obs", obs)
else:
diagnostics().push("test_obs", np.nan)
# store data in wrf_file variable FMC_G
if cfg['write_fields']:
ncfmc_gc[t,:Nk,:,:] = np.transpose(models.get_state()[:,:,:Nk],axes=[2,0,1])
# store the diagnostics in a binary file when done
diagnostics().dump_store(os.path.join(cfg['output_dir'], 'diagnostics.bin'))
# close the netCDF file (relevant if we did write into FMC_GC)
if out_file is not None:
out_file.close()
