class eoldas_setup(object):
"""
This is redundant
"""
def __init__(self,datafile,options):
self.options = options
# sort logging
self.options.general.logdir \
= self.__getopt(self.options.general,'logdir',"logs")
self.options.general.logfile \
= self.__getopt(self.options.general,'logfile',"logfile.log")
# 1.- set up logging for this particular run
self.logger = set_up_logfile(self.options.general.logfile,\
name="eoldas_setup",logdir=self.options.general.logdir)
try:
self.setup()
except:
self.logger.error("Unable to access critical elements of the options. See help(eoldas_setup.setup) for details")
sys.exit(-1)
# read conf file(s)
self.configfile = data_file
#config = ConfFile(self.configfile,dirs=dirs,log_name=self.logger)
if len(config.infos) == 0:
self.fail = True
return
# not sure what to do if multiple config files???
# just take the first one at the moment
self.config = config.infos[0]
# update with cmd line options
self.config.update(self.options,combine=True)
self.logger.info("Model sd scaling by %f over that defined in the config file" % self.config.general.model_sd)
self.ok = self.process_config_file()
def setup(self):
'''
Access and set up critical elements of the problem.
This includes:
The existence of:
options.parameter.names (a list)
'''
options = self.options
options.parameter.n_params = len(options.parameter.names)
self.options = options
def __getopt(self,options,key,value):
if not hasattr(options,key):
options[key] = value
return options[key]
def __pcheck(self,thisdict,name):
'''
Check that name exists in thisdict
'''
try:
if name in thisdict.dict():
return True
else:
return False
except:
return False
def __min(self,a,b):
'''
Min utility for 2 numbers, ignoring None
'''
if a == None:
out = b
elif b == None:
out = a
else:
out = np.min([a,b])
if out == None:
return 0
else:
return out
# the next critical thing is some observations
obs = load_brdf_file (brf,self.config,bandpass_names={})
if obs == False:
return False
self.config.operator.obs.update(obs,combine=True)
# sets up an initial version of x_init
# which is in the observation 'space' (ie one per obs)
for n_par in xrange ( self.config.params.n_params ):
#self.default_vals[n_par] = prior_mean[n_par]
if np.all( self.obs.x_init[ :, n_par] == 0 ): # No
self.obs.x_init [ :, n_par ] = self.default_vals [ n_par ]
# try brfinit_files
# which can overwrite x_init
try:
if self.options.preload != []:
brfinit_files = self.options.preload
self.brfinit_files['override'] = brfinit_files
except:
if self.options.preload != []:
self.brfinit_files = ParamStorage ()
self.brfinit_files['override'] = self.options.preload
# this is a hack to get the same structure
self.brfinit_files = self.brfinit_files.dict()
thisdoys = None
if self.brfinit_files is not None:
# this is not consistent with having multiple files
# and is a bit of a mess
for key in self.brfinit_files.keys():
if type(self.brfinit_files[key]) == type([]):
initfile = self.brfinit_files[key][0]
else:
initfile = self.brfinit_files[key]
#(acovar, abandwidth, abands, anpt, anbands_max, alocation, \
# awhichfile, anbands, adoys, aqa, atheta_v, atheta_i,aphi_v, \
# aphi_i, aisobs, aobs, aobscovar, aparams_x) = \
# load_brdf_file(initfile)
(thisdoys,thisparams) = self.read_parameters(initfile,confdir=confdir)
# if fail, thisdoys is None
#self.obs.x_init[:,:] = aparams_x[:,:]
if thisdoys == None:
self.brfinit_files = None
# For convenience, we can invert the observation covariance matrices
self.obs.obsinvcovar = []
self.obs.real_obsinvcovar = []
for sample_no in xrange( self.obs.npt ):
temp_mtx = np.matrix( self.obs.obscovar[ sample_no ] ).I
if self.config.params.scale_cost:
self.logger.info ("Scaling obs by %f" % \
float(self.obs.npt*self.obs.nbands[0] ) )
self.obs.obsinvcovar.append ( \
temp_mtx/float((self.obs.npt*self.obs.nbands[sample_no] )))
else:
self.obs.obsinvcovar.append( temp_mtx )
self.obs.real_obsinvcovar.append (temp_mtx)
# if there is anything non zero in x_init, set params_x to that
if self.obs.x_init.sum() > 0:
self.params_x = self.obs.x_init.copy()
else:
self.params_x = np.zeros ((self.obs.npt, \
self.config.params.n_params))
# determine which params to fix, based primarily on solve_for flags
fix_params = define_fixparams(self.parameters, \
solve_for=self.solve_for,prior_sd=self.prior_sd,model_unc_cfg=self.model_unc_cfg)
self.config.params.n_model_params = np.sum(fix_params==3) + np.sum(fix_params==4)
# set up the grid based on the span of unique doys
self.unique_doys, self.quantised_doys, self.obs_shift = quantise_time ( self.obs.doys, \
self.time_quant ,grid=grid)
self.grid_n_obs = self.unique_doys.shape[0]
self.fix_params = np.tile(fix_params, self.grid_n_obs).reshape((self.grid_n_obs,self.config.params.n_params))
self.logger.info ("%d days, %d quantised days" % ( len(self.unique_doys), \
len(self.quantised_doys) ) )
self.grid_n_params = fix_params.shape[0]
# set up a grid model representation from self.params_x
# we will use then when loading
# self.params_x is a full representation in obs space
# so we expand it to the model grid space
self.store_params = self.get_x(self.params_x,self.fix_params*0.)
# but this may contain zeros if a parameter has not been defined so should be set to the default value
# or maybe interpolations is better
udoys = np.unique(self.obs.doys)
try:
where_udoys = np.in1d(self.unique_doys,udoys)
except:
where_udoys = np.zeros_like(self.unique_doys).astype(np.bool)
for i in udoys:
w = np.where(self.unique_doys == i)
where_udoys[w] = True
for i in xrange(self.grid_n_params):
self.store_params[:,i] = np.interp(self.unique_doys,self.unique_doys[where_udoys],self.store_params[where_udoys,i])
# override this with data from brfinit_files
if self.brfinit_files is not None:
# zeroth ...
# pull out elements of thisdoys that appear in self.unique_doys
# first interpolate thisparams onto the grid
store_params = self.store_params*0.
new_thisdoys = np.zeros( self.store_params.shape[0]).astype(np.int)
# loop over thisdoys and load where appropriate
for (i,j) in enumerate(thisdoys):
ww = np.where(j == self.unique_doys)
store_params[ww,:] = thisparams[i,:]
new_thisdoys[ww] = j
thisdoys = new_thisdoys
udoys = np.unique(thisdoys)
try:
where_udoys = np.in1d(thisdoys,udoys)
except:
where_udoys = np.zeros_like(thisdoys).astype(np.bool)
for i in udoys:
w = np.where(where_udoys == i)
where_udoys[w] = True
for i in xrange(self.grid_n_params):
self.store_params[:,i] = np.interp(self.unique_doys,self.unique_doys[where_udoys],store_params[where_udoys,i])
# deal with model uncert
self.model_unc = np.ones((self.fix_params.shape[1]))
for ( i, k ) in enumerate ( self.parameters ):
if self.model_unc_cfg [ k ] > 0:
self.model_unc[i] = self.model_unc[i] * self.model_unc_cfg [ k ]
self.prior_m = np.array([self.prior_mean[k] for k in self.parameters ])
self.prior_std = np.array([self.prior_sd[k] for k in self.parameters ])
return #( prior_mean, prior_sd, model_unc, abs_tol, scale_cost)
def get_x( self, x_obs, x_model, summer=False):
"""
return an x_model representation which has parameter values for the
complete model grid. The array x_obs has a representation of the
parameter values only for observation points, whereas x_model is
typically defined over the whole assimilation period/region.
When loading parameters in this way (from observation space
to model space, only the parameter associated with the first
observation at a particular point is taken (summer=False)
When loading derivatives (e.g. when using the adjoint) we need
to sum over all observation grid points (summer=True)
Parameters
-----------
x_obs : array-like
The state vector representation that corresponds to the observations
x_model : array-like
The state vector representation that corresponds to the assimilation
interval.
"""
if summer == False:
for i in np.unique(self.obs_shift).astype(np.int):
w = np.where(self.obs_shift == i)[0][0]
x_model[i,:] = x_obs[w,:]
else:
x_model[:,:] = 0.
for i in np.unique(self.obs_shift).astype(np.int):
w = np.where(self.obs_shift == i)[0]
for j in w:
x_model[i,:] = x_model[i,:] + x_obs[j,:]
return x_model
def write_parameters(self,filename,params,ofmt='ASCII'):
"""
Write the parameters out to filename
"""
if ofmt == 'ASCII':
self.logger.info ( "Saving parameters to %s" % filename)
fp = open(filename,'w')
fp.write("# PARAMETERS %s\n" % "".join ( [ "%s " % i for i in self.parameters]))
for i in xrange(self.grid_n_obs):
fp.write("%f %s\n" % (self.unique_doys[i],"".join ( [ "%s " % j for j in params[i,:]])))
fp.close()
