def __init__(self,argv,name='eoldas',logger=None): from eoldas.eoldas_Lib import sortopt, sortlog argv = argv or sys.argv here = os.getcwd() self.thisname = name Parser.__init__(self,argv,name=self.thisname,logger=logger,\ general=None,outdir=".",getopdir=False,parse=True) os.chdir(here) if not hasattr(self,'configs'): self.logger.error('No configration file specfied') help(eoldas) return self.thisname = name solver = eoldas_Solver(self,logger=self.logger,name=self.thisname+'.solver') self.general = sortopt(self.root[0],'general',ParamStorage()) self.general.write_results = sortopt(self.general,'write_results',True) self.general.calc_posterior_unc = sortopt(self.general,'calc_posterior_unc',False) self.general.passer = sortopt(self.general,'passer',False) self.solver = solver self.logger.info('testing full cost functions') for i in xrange(len(solver.confs.infos)): self.logger.info('%d/%d ...'%(i+1,len(solver.confs.infos))) # try an initial solver.prep(i) J = solver.cost(None) J_prime = solver.cost_df(None) self.logger.info('done') # give the user some info on where the log file is # in case theyve forgotten print 'logging to',self.general.logfile
def __unload(self, options): from eoldas_ConfFile import array_type_convert this = ParamStorage() this.general = ParamStorage() for (k, v) in options.iteritems(): ps = this that = k.split('.') if len(that) == 1: ps = this.general else: for i in xrange(len(that) - 1): if not hasattr(ps, that[i]): ps[that[i]] = ParamStorage() ps = ps[that[i]] # set the value v which needs to # to be interpreted ps[that[-1]] = array_type_convert(self.top, v) return this
def read_numpy_fromfile(self, goodfile, dataset, info=None): ''' Utility to try to read a file goodfile This simple version of the utility just tries a numpy.fromfile It is not generally recommended to use this first as it will skip headers etc that may contain information However, this is a good illustration of the required interface format for file readers. Inputs: goodfile : a filename (ideally one that exists) dataset : dataset name (e.g. x_state) info : a list of other information Outputs: retval : ParamStorage error : tuple where: retval contains dataset 'this' in data['this'] and other information (e.g. locations etc) in retval.data and retval.names ''' #self.store_header = goodfile.open().readline().close() this = np.genfromtxt(goodfile, skip_header=1) l = len(np.atleast_1d(this).shape) if this.size > 0: retval = ParamStorage() retval.name = ParamStorage() retval.data = ParamStorage() retval.data[dataset] = this retval.name.filename = goodfile retval.name.fmt = 'np.genfromtxt' return retval, (False, "") error = True error_msg = "Failed numpy read of %s" % goodfile return 0, (error, error_msg)
def parseLoader(self, loaders): """ Utility to load a set of terms from the list loaders into the ParamStorage general If there are 3 terms in each loaders element, they refer to: 1. name 2. default value 3. helper text If there is a fourth, it is associated with extras (short parser option) """ general = ParamStorage() general.__default__ = ParamStorage() general.__extras__ = ParamStorage() general.__helper__ = ParamStorage() for this in loaders: if len(this) > 1: general[this[0]] = this[1] general.__default__[this[0]] = this[1] else: general[this[0]] = None general.__default__[this[0]] = None if len(this) > 2: general.__helper__[this[0]] = this[2] else: general.__helper__[this[0]] = optparse.SUPPRESS_HELP if len(this) > 3: general.__extras__[this[0]] = "%s" % this[3] else: general.__extras__[this[0]] = None # make sure arrays arent numpy.ndarray if type(general.__default__[this[0]]) == np.ndarray: general.__default__[this[0]] = \ list(general.__default__[this[0]]) self.top.update(self.__unload(general), combine=True)
def safesplit(text, character): ''' A function to split a string, taking account of [] and () and quotes ''' lent = len(text) intoken = ParamStorage() qtoken = ParamStorage() for i in ["'", '"']: qtoken[i] = False for i in ["()", "[]", "{}"]: intoken[i] = 0 start = 0 lst = [] i = 0 while i < lent: # are any of intoken, qtoken open isopen = False for (j, k) in intoken.iteritems(): if text[i] == j[0]: intoken[j] += 1 elif text[i] == j[1]: intoken[j] -= 1 isopen = isopen or (intoken[j] != 0) for (j, k) in qtoken.iteritems(): if text[i] == j[0]: qtoken[j] = not qtoken[j] isopen = isopen or qtoken[j] if text[i] == character and not isopen: lst.append(text[start:i]) start = i + 1 elif text[i] == '\\': i += 2 continue i += 1 lst.append(text[start:]) return lst
def sortlog(self,logfile,logger,name="eoldas",logdir=None,debug=True ): ''' A safe interface to logging for passing log information between lots of classes ''' import logging import time from eoldas_Lib import set_up_logfile,dummyprint if type(self).__name__ == 'SpecialVariable' and logger != None: return logger try: if 'logger' in self.dict(): return self.logger except: pass try: if 'logger' in self.keys(): return self.logger except: pass try: if type(self).__name__ != 'SpecialVariable': this = self.logger return self.logger except: pass if logdir == None: logdir = '.' if name == None: name = type(self).__name__ + '.' + str(time.time()) if logger: if type(self).__name__ == 'SpecialVariable': return logger logger.info('Setting up logger for %s'%name) logger = logging.getLogger(name) logger.info('Set up') return logger if logfile == None or name == None: logger = ParamStorage() logger.info = lambda x:dummyprint( "Info: %s"%x) logger.debug = lambda x:dummyprint( "Debug: %s"%x) logger.error = lambda x:dummyprint( "Error: %s"%x) return logger logger = set_up_logfile(logfile,name=name,logdir=logdir,debug=debug) return logger
def __init__ ( self, nbands, nbands_max, npt, bandwith, obscovar, \ location, whichfile, doys, qa, theta_v, theta_i, \ phi_v, phi_i, isobs, params_x, \ obs=0.0, bandpass_library=False ) : # Define a configuration container. Makes everything look like Java # Containers can then be useful for quickly listing all variables :) self.config = ParamStorage () self.config.spectral = ParamStorage () self.config.rt_model = ParamStorage () self.observations = ParamStorage () self.config.rt_model.nparams = rt_getnparams () self.config.npt = npt # Number of points self.config.nv = 1 # Always set to 1. No questions asked self._setup_spectral_config ( nbands, nbands_max, bandwith ) self._setup_rt_model () self._setup_bandpass_funcs ( bandwidth, bandpass_library ) self._setup_geometry ( theta_v, phi_v, theta_i, phi_i ) self.params_x = np.zeros (( self.config.npt, \ self.config.rt_model.nparams ) ) self.observations.brf = np.zeros([self.config.npt, \ self.config.spectral.nbands_max] ) self.observations.obs = np.zeros([self.config.npt, \ self.config.spectral.nbands_max] ) # obs will be set to zero if not loaded self.observations.obs[:,:] = obs # setting self.brf_ad to 1 means that we calculate the model derivative # by default self.brf_ad = np.ones([self.config.npt, \ self.config.spectral.nbands_max] ) self._set_minmax ( ) self._set_x( x )
def demonstration(conf='default.conf'): ''' Need to develop a new demo ''' from eoldas_Lib import eoldas_setup from os import getcwd from eoldas_ParamStorage import ParamStorage options = ParamStorage() options.here = getcwd() options.logdir = 'logs' options.logfile = "logfile.log" options.datadir = ['.','~/.eoldas'] #self = eoldas_setup('default.conf',options) #return self return True
def startlog(self, log_terms, name=None): ''' Start the logger. This is called on initialisation and you shouldn't normally need to access it. ''' import logging from eoldas_Lib import set_up_logfile try: self.logger.shutdown() except: self.logger = ParamStorage() logfile = log_terms['logfile'] or self.options.logfile logdir = log_terms['logdir'] or self.options.logdir name = name or self.options.thisname self.logger = set_up_logfile(logfile,\ name=name,logdir=logdir)
def setup_rt_model(self): """ This sets up the RT model (and adjoint if available) by calling any preparation methods. """ if not 'linear' in self.dict(): self.linear = ParamStorage() if 'y' in self.dict(): self.linear.H = np.zeros(self.y.state.shape) else: self.linear.H = np.zeros(self.x.state.shape) self.nv = 1 self.npt = len(self.y.state) self.linear.J_prime = np.zeros(self.x.state.shape) if not self.rt_model.use_median: self.bandIndex = self.y_meta.spectral.all_bands else: self.bandIndex = self.y_meta.spectral.median_bands self.linear.brf_ad = np.ones((self.npt, len(self.bandIndex))) self.rt_library.rt_modelpre(np.array(self.bandIndex) + 1) self.rt_library.rt_modeldpre(self.npt) self.x_orig = self.x.state.copy() if self.rt_model.use_median: bands_to_use = self.y_meta.spectral.median_bands_to_use bandpass_library = self.y_meta.spectral.median_bandpass_library index = self.y_meta.spectral.median_bandpass_index else: bands_to_use = self.y_meta.spectral.bands_to_use bandpass_library = self.y_meta.spectral.bandpass_library index = self.y_meta.spectral.bandpass_index self.y_meta.spectral.bands_to_use = \ np.zeros((len(bands_to_use),len(self.bandIndex))) for (i, bandname) in enumerate(self.y_meta.spectral.bandnames): fullb = bandpass_library[bandname] this = fullb[index[bandname]] this = this / this.sum() ww = np.where(np.in1d(self.bandIndex, index[bandname]))[0] self.y_meta.spectral.bands_to_use[i, ww] = this
def scan_info(self, config, this, info, fullthis, fullinfo): """ Take a ConfigParser instance config and scan info into config.info. This is called recursively if needed. Parameters: config : the configuration object this : the current item to be parsed info : where this item is to go fullthis : the full name of this fullinfo : the full (top level) version of info. """ from eoldas_ConfFile import assoc_to_flat # find the keys in the top level # loop over thiss = np.array(this.split('.')) # just in case .. is used as separator ww = np.where(thiss != '') thiss = thiss[ww] nextone = '' for i in xrange(1, len(thiss) - 1): nextone = nextone + thiss[i] + '.' if len(thiss) > 1: nextone = nextone + thiss[-1] # first, check if its already there if not hasattr(info, thiss[0]): info[thiss[0]] = ParamStorage() info[thiss[0]].helper = [] # load up the info if len(thiss) == 1: for option in config.options(fullthis): fulloption = option # option may have a '.' separated term as well options = np.array(option.split('.')) # tidy up any double dot stuff ww = np.where(options != '') options = options[ww] # need to iterate to make sure it is loaded # at the right level # of the hierachy this_info = info[this] # so now this_info is at the base for i in xrange(len(options) - 1): if not hasattr(this_info, options[i]): this_info[options[i]] = ParamStorage() this_info[options[i]].helper = [] this_info = this_info[options[i]] option = options[-1] this_info[option] = array_type_convert(fullinfo,\ config.get(fullthis,fulloption)) if option[:6] == 'assoc_': noption = option[6:] this_info[noption] = assoc_to_flat(\ fullinfo.parameter.names,this_info[option],\ this_info[noption]) is_assoc = True else: is_assoc = False if not hasattr(this_info, 'helper'): this_info.helper = [] ndot = len(fullthis.split('.')) pres = '' for i in xrange(1, ndot): pres += ' ' if type(this_info.helper) == str: this_info.helper += "\n%s%s.%-8s = %-8s" % \ (pres,fullthis,fulloption,str(this_info[option])) elif type(this_info.helper) == list: this_info.helper.append("%s%s.%-8s = %-8s" % \ (pres,fullthis,fulloption,\ str(this_info[option]))) if is_assoc: if type(this_info.helper) == str: this_info.helper += "\n%s%s.%-8s = %-8s" % \ (pres,fullthis,fulloption.replace\ ('assoc_',''),str(this_info[noption])) elif type(this_info.helper) == list: this_info.helper.append("%s%s.%-8s = %-8s" % \ (pres,fullthis,fulloption.replace\ ('assoc_',''),str(this_info[noption]))) else: self.scan_info(config, nextone, info[thiss[0]], fullthis, fullinfo) if thiss[-1][:6] == 'assoc_' and thiss[0] in fullinfo.dict(): # only do this operation when at the top level noption = thiss[-1][6:] option = thiss[-1] this_info = info fulloption = thiss[0] this_info = this_info[thiss[0]] for i in xrange(1, len(thiss) - 1): this_info = this_info[thiss[i]] fulloption = '%s.%s' % (fulloption, thiss[i]) fulloption = '%s.%s' % (fulloption, noption) #this_info[noption] = assoc_to_flat(fullinfo.parameter.names\ # ,this_info[option],\ # this_info[noption]) if not 'names' in this_info.dict(): this_info.names = fullinfo.parameter.names if not option in this_info.dict(): this_info[option] = [0] * len(this_info.names) if not noption in this_info.dict(): this_info[noption] = [0] * len(this_info.names) this_info[noption] = assoc_to_flat(this_info.names\ ,this_info[option],\ this_info[noption]) ndot = len(fullthis.split('.')) pres = '' for i in xrange(1, ndot): pres += ' ' if type(this_info.helper) == str: this_info.helper += "\n%s%-8s = %-8s" % (pres,\ fulloption,str(this_info[noption])) elif type(this_info.helper) == list: this_info.helper.append("%s%-8s = %-8s" % (pres,\ fulloption,str(this_info[noption])))
def demonstration(): from eoldas_State import State from eoldas_ParamStorage import ParamStorage import numpy as np # a basic set up for State, setting names & bounds etc options = ParamStorage() options.logfile = 'test/data_type/logs/log.dat' options.names = \ 'gamma xlai xhc rpl xkab scen xkw xkm xleafn xs1 xs2 xs3 xs4 lad'.split() options.bounds = [[0.01,None],\ [0.01,0.99],\ [0.01,10.0],\ [0.001,0.10],\ [0.1,0.99],\ [0.0,1.0],\ [0.01,0.99],\ [0.3,0.9],\ [0.9,2.5],\ [0.0, 4.],\ [0.0, 5.],\ [None, None],\ [None, None],\ [None, None]] options.default = -1.0 * np.ones(len(options.names)) options.location = 'time'.split() options.control = 'mask vza vaa sza saa'.split() options.datadir = ['.', 'test/data_type'] name = "eoldas_data_type test 0" options.limits = [[170, 365, 1]] self = State(options,datatype='y',name=name,datadir=\ options.datadir,env=None,logfile=options.logfile) self.tester() # Now we set some state data this = ParamStorage() # how many state vector elements should there be? n_states = len(self.Name.state) self.state = np.ones([100, n_states]) self.Data.sd = np.ones([1, n_states]) self.Name.sd = self.Name.state print '******************' print(self.Data.sd, self.Name.sd) this.data = ParamStorage() this.name = ParamStorage() this.data.state = self.state * 2 controls = self.Name.control n_controls = len(controls) this.data.control = np.ones([100, n_controls]) this.data.location = np.ones([100, n_controls]) # we can load x_state from a ParamStorage self.state = this # now we should see the control data etc. self.tester() # change a dataset name to see if that works: # should load everything as numpy arrays self.Name.control = np.array(['vza', 'sza']) # change a dataset to see if that works .. deliberately load a bad one self.Data.control = 0 # now try a direct state data load self.state = np.zeros([100, 100]) + 5. # which will load into self.Data.state self['state'] = np.zeros([100, 100]) + 6. #now try accessing it: print self.state # change the control info self.Name.control = np.array(['vza']) print self.Name.control # reset it self.state = self.state * 2 print self.state # now try reading a file into state self.state = 'test/data_type/input/test.brf' print '========' print 'data from a BRDF file' self.tester() print '========' # written as a pickle self.write('test/data_type/output/test.pickle', None, fmt='pickle') self.logger.info("...DONE...") name = "eoldas_data_type test 1" del self self1 = State(options,datatype='y',name=name,datadir=\ options.datadir,env=None,logfile=options.logfile,grid=True) # read from pickle self1.state = 'test/data_type/output/test.pickle' print '========' print 'data from a pickle file' self1.tester() print '========' # try to load an npx file del self1 options.location = 'time row col'.split() options.limits = [[170, 365, 1], [0, 500, 1], [200, 200, 1]] self2 = State(options,datatype='y',name=name,datadir=\ options.datadir,env=None,logfile=options.logfile) self2.state = 'test/data_type/input/interpolated_data.npz' print '========' print 'data from a npz file' self2.tester() print '========' # write as BRDF-UCL self2.write('test/data_type/output/test.brf', None, fmt='BRDF-UCL') del self2 self3 = State(options,datatype='y',name=name,datadir=\ options.datadir,env=None,logfile=options.logfile) # then test the reader self3.state = 0. self3.state = 'test/data_type/output/test.brf' print '========' print 'data from a BRDF-UCL file' print '========' self3.tester() print '========' # then write as a PARAMETERS file self3.write('test/data_type/output/test.param', None, fmt='PARAMETERS') del self3 options.location = 'time row col'.split() options.limits = [[170, 365, 1], [0, 500, 1], [200, 200, 1]] options.control = np.array(['mask', 'vza', 'vaa', 'sza', 'saa']) self4 = State(options,datatype='y',name=name,datadir=\ options.datadir,env=None,logfile=options.logfile) # then test the reader self4.state = 0. self4.state = 'test/data_type/output/test.param' print '========' print 'data from a PARAMETERS file' print '========' self4.tester() print '========'
def __min(self,a,b): ''' Min utility for 2 numbers, ignoring None ''' if a == None: out = b elif b == None: out = a else: out = np.min([a,b]) if out == None: return 0 else: return out # the next critical thing is some observations obs = load_brdf_file (brf,self.config,bandpass_names={}) if obs == False: return False self.config.operator.obs.update(obs,combine=True) # sets up an initial version of x_init # which is in the observation 'space' (ie one per obs) for n_par in xrange ( self.config.params.n_params ): #self.default_vals[n_par] = prior_mean[n_par] if np.all( self.obs.x_init[ :, n_par] == 0 ): # No self.obs.x_init [ :, n_par ] = self.default_vals [ n_par ] # try brfinit_files # which can overwrite x_init try: if self.options.preload != []: brfinit_files = self.options.preload self.brfinit_files['override'] = brfinit_files except: if self.options.preload != []: self.brfinit_files = ParamStorage () self.brfinit_files['override'] = self.options.preload # this is a hack to get the same structure self.brfinit_files = self.brfinit_files.dict() thisdoys = None if self.brfinit_files is not None: # this is not consistent with having multiple files # and is a bit of a mess for key in self.brfinit_files.keys(): if type(self.brfinit_files[key]) == type([]): initfile = self.brfinit_files[key][0] else: initfile = self.brfinit_files[key] #(acovar, abandwidth, abands, anpt, anbands_max, alocation, \ # awhichfile, anbands, adoys, aqa, atheta_v, atheta_i,aphi_v, \ # aphi_i, aisobs, aobs, aobscovar, aparams_x) = \ # load_brdf_file(initfile) (thisdoys,thisparams) = self.read_parameters(initfile,confdir=confdir) # if fail, thisdoys is None #self.obs.x_init[:,:] = aparams_x[:,:] if thisdoys == None: self.brfinit_files = None # For convenience, we can invert the observation covariance matrices self.obs.obsinvcovar = [] self.obs.real_obsinvcovar = [] for sample_no in xrange( self.obs.npt ): temp_mtx = np.matrix( self.obs.obscovar[ sample_no ] ).I if self.config.params.scale_cost: self.logger.info ("Scaling obs by %f" % \ float(self.obs.npt*self.obs.nbands[0] ) ) self.obs.obsinvcovar.append ( \ temp_mtx/float((self.obs.npt*self.obs.nbands[sample_no] ))) else: self.obs.obsinvcovar.append( temp_mtx ) self.obs.real_obsinvcovar.append (temp_mtx) # if there is anything non zero in x_init, set params_x to that if self.obs.x_init.sum() > 0: self.params_x = self.obs.x_init.copy() else: self.params_x = np.zeros ((self.obs.npt, \ self.config.params.n_params)) # determine which params to fix, based primarily on solve_for flags fix_params = define_fixparams(self.parameters, \ solve_for=self.solve_for,prior_sd=self.prior_sd,model_unc_cfg=self.model_unc_cfg) self.config.params.n_model_params = np.sum(fix_params==3) + np.sum(fix_params==4) # set up the grid based on the span of unique doys self.unique_doys, self.quantised_doys, self.obs_shift = quantise_time ( self.obs.doys, \ self.time_quant ,grid=grid) self.grid_n_obs = self.unique_doys.shape[0] self.fix_params = np.tile(fix_params, self.grid_n_obs).reshape((self.grid_n_obs,self.config.params.n_params)) self.logger.info ("%d days, %d quantised days" % ( len(self.unique_doys), \ len(self.quantised_doys) ) ) self.grid_n_params = fix_params.shape[0] # set up a grid model representation from self.params_x # we will use then when loading # self.params_x is a full representation in obs space # so we expand it to the model grid space self.store_params = self.get_x(self.params_x,self.fix_params*0.) # but this may contain zeros if a parameter has not been defined so should be set to the default value # or maybe interpolations is better udoys = np.unique(self.obs.doys) try: where_udoys = np.in1d(self.unique_doys,udoys) except: where_udoys = np.zeros_like(self.unique_doys).astype(np.bool) for i in udoys: w = np.where(self.unique_doys == i) where_udoys[w] = True for i in xrange(self.grid_n_params): self.store_params[:,i] = np.interp(self.unique_doys,self.unique_doys[where_udoys],self.store_params[where_udoys,i]) # override this with data from brfinit_files if self.brfinit_files is not None: # zeroth ... # pull out elements of thisdoys that appear in self.unique_doys # first interpolate thisparams onto the grid store_params = self.store_params*0. new_thisdoys = np.zeros( self.store_params.shape[0]).astype(np.int) # loop over thisdoys and load where appropriate for (i,j) in enumerate(thisdoys): ww = np.where(j == self.unique_doys) store_params[ww,:] = thisparams[i,:] new_thisdoys[ww] = j thisdoys = new_thisdoys udoys = np.unique(thisdoys) try: where_udoys = np.in1d(thisdoys,udoys) except: where_udoys = np.zeros_like(thisdoys).astype(np.bool) for i in udoys: w = np.where(where_udoys == i) where_udoys[w] = True for i in xrange(self.grid_n_params): self.store_params[:,i] = np.interp(self.unique_doys,self.unique_doys[where_udoys],store_params[where_udoys,i]) # deal with model uncert self.model_unc = np.ones((self.fix_params.shape[1])) for ( i, k ) in enumerate ( self.parameters ): if self.model_unc_cfg [ k ] > 0: self.model_unc[i] = self.model_unc[i] * self.model_unc_cfg [ k ] self.prior_m = np.array([self.prior_mean[k] for k in self.parameters ]) self.prior_std = np.array([self.prior_sd[k] for k in self.parameters ]) return #( prior_mean, prior_sd, model_unc, abs_tol, scale_cost)
def read_numpy(self, filename, name, info=[]): ''' Try to read the file as as a NpzFile file ''' from eoldas_Lib import set_default_limits,check_limits_valid,\ quantize_location,dequantize_location # none of these ciritical to functioning try: info = self._state.info except: info = [] try: names = self.name.state except: try: names = self.Name.state except: names = None try: control = self.Name.control except: try: control = self.name.control except: control = None try: location = self.name.location except: try: location = self.Name.location except: location = ['time', 'row', 'col'] try: limits = self.name.qlocation except: try: limits = self.Name.qlocation except: limits = set_default_limits(location) # refl_check=False,names=None,\ # control=['mask','vza','vaa','sza','saa'],\ # location=['time','row','col'],limits=None # location specifies the dimesions and names of the # problem, e.g., & typically [time,row,col] limits = np.array(check_limits_valid(limits)) try: f = np.load(filename) if not type(f).__name__ == 'NpzFile': f.close() self.error_msg = "%s is not a NpzFile" % filename self.error = True if 'logger' in self or 'logger' in self.dict(): self.logger.info(self.error_msg) return 0, (self.error, self.error_msg) except: self.error_msg = "a problem opening %s as a NpzFile" % filename self.error = True if 'logger' in self or 'logger' in self.dict(): self.logger.info(self.error_msg) return 0, (self.error, self.error_msg) # ok so far then # lets have a look inside ncontents = np.array(f.files) contents = np.array(f.files) # translation table for default names def_names = 'b1 b2 b3 b4 b5 b6 b7'.split() if names == None: # assume MODIS names = def_names def_alt_names = \ '645.5 856.5 465.6 553.6 1241.6 1629.1 2114.1'.split() # look for any of names in contents datasets = [] alt_datasets = [] alt_names = names for i in xrange(len(np.atleast_1d(contents))): if contents[i] in names: datasets.append(i) if not len(np.atleast_1d(datasets)): if 'logger' in self or 'logger' in self.dict(): self.logger.error(\ "None of requested datasets %s found in %s ..." \ %(str(names),filename) + \ " trying default MODIS names: only %s"\ %(str(contents))) names = def_names alt_names = def_alt_names for i in xrange(len(np.atleast_1d(contents))): if contents[i] in names: datasets.append(i) if not len(np.atleast_1d(datasets)): self.error_msg = "None of requested datasets %s found in %s"\ %(str(names),filename) + ' ' + \ "... trying default MODIS names: only %s"\ %(str(contents)) self.error = True if 'logger' in self or 'logger' in self.dict(): self.logger.error(self.error_msg) return 0, (self.error, self.error_msg) trans_names = {} for (i, j) in enumerate(alt_names): trans_names[names[i]] = j #trans_names = {names[i]:j for (i,j) in enumerate(alt_names)} alt_name = [] this_name = [] for i in datasets: this_name.append(contents[i]) alt_name.append(trans_names[contents[i]]) # Translate some old stylies... trans = {'raa': 'vaa', 'doys': 'time'} for i in trans: if i in contents: ncontents[np.where(contents == i)[0]] = trans[i] # as a minimum, there needs to be some definition of one of # the terms in location # check how many dimensions this has # now find a dataset try: # This could be more general, but this will do for now as its useful # for spatial datasets QA_OK = np.array(\ [8, 72, 136, 200, 1032, 1288, 2056,2120, 2184, 2248]) doy = f['doys'] - 2004000 qa = f['qa'] vza = f['vza'] sza = f['sza'] raa = f['raa'] y = [] for i in this_name: y.append(f[i]) #mask = np.logical_or.reduce([qa==x for x in QA_OK ]) if 'logger' in self or 'logger' in self.dict(): self.logger.info(\ "sucessfully interpreted NpzFile dataset from %s"\ %filename) self.logger.info("sub-setting ...") controls = [] locations = [] grid = [] qlocations = [] thisshape = vza.shape starter = {'time': np.min(doy), 'row': 0, 'col': 0} delta = {'time': 1, 'row': 1, 'col': 1} if len(np.atleast_1d(limits)) < 3: from eoldas_Lib import set_default_limits old_loc = location location = np.array(['time', 'row', 'col']) lim2 = set_default_limits(location) for i in xrange(len(np.atleast_1d(limits))): ww = np.where(old_loc[i] == location)[0] lim2[ww] = list(limits[i]) limits = lim2 for i in xrange(len(np.atleast_1d(limits))): if limits[i][0] == None: limits[i][0] = starter[location[i]] if limits[i][1] == None: limits[i][1] = (thisshape[i] - 1) + starter[location[i]] if limits[i][2] == None: limits[i][2] = delta[location[i]] limits = np.array(limits) start_doy = limits[0][0] end_doy = limits[0][1] step_doy = limits[0][2] start_row = limits[1][0] end_row = limits[1][1] step_row = limits[1][2] start_col = limits[2][0] end_col = limits[2][1] step_col = limits[2][2] gooddays = np.logical_and.reduce(np.concatenate(\ ([doy >= start_doy],[doy <=end_doy]))) qa = qa[gooddays, start_row:end_row + 1, start_col:end_col + 1] vza = vza[gooddays, start_row:end_row + 1, start_col:end_col + 1] * 0.01 sza = sza[gooddays, start_row:end_row + 1, start_col:end_col + 1] * 0.01 raa = raa[gooddays, start_row:end_row + 1, start_col:end_col + 1] * 0.01 yy = [] for i in xrange(len(np.atleast_1d(this_name))): this = y[i] yy.append(this[gooddays,start_row:end_row+1,\ start_col:end_col+1]*0.0001) doy = doy[gooddays] # now do QA mask = np.zeros_like(qa).astype(bool) # loop over qa for j in xrange(len(np.atleast_1d(QA_OK))): ww = np.where(qa == QA_OK[j]) mask[ww] = True # better look over data to check valid for j in xrange(len(np.atleast_1d(yy))): ww = np.where(yy[j] < 0) mask[ww] = False ww = np.where(mask) if 'logger' in self or 'logger' in self.dict(): self.logger.debug('parsing dataset: %d samples look ok'\ %np.array(ww).shape[1]) vza = vza[ww] sza = sza[ww] raa = raa[ww] doy = doy[ww[0]] row = ww[1] + start_row col = ww[2] + start_col locations = np.array([doy, row, col]) nnn = len(np.atleast_1d(locations[0])) orig = np.repeat(np.array([start_doy, start_row, start_col]), locations.shape[1]).reshape(locations.shape).T div = np.repeat(np.array([step_doy, step_row, step_col]), locations.shape[1]).reshape(locations.shape).T qlocations = ((locations.T - orig) / div.astype(float)).astype(int).T controls = np.array([np.ones_like(doy).astype(bool),\ vza,raa,sza,0*doy]) y = [] for i in xrange(len(np.atleast_1d(this_name))): this = yy[i] y.append(this[ww]) grid = np.array(y) fmt = 'BRDF-UCL' control = ['mask', 'vza', 'vaa', 'sza', 'saa'] bands = alt_name if not np.array(grid).size: if 'logger' in self or 'logger' in self.dict(): self.logger.error(\ "Warning: returning a zero-sized dataset ... "+\ " I wouldn;t try to do anything with it") # in case we dont have data for all bands mask = np.logical_or.reduce([[this_name[i]==x for x in names] \ for i in xrange(len(np.atleast_1d(this_name)))]) sd = np.array('0.004 0.015 0.003 0.004 0.013 0.01 0.006'\ .split())[mask] sd = np.array([float(i) for i in sd.flatten()])\ .reshape(sd.shape) nsamps = grid.shape[1] sd = sd.repeat(nsamps).reshape(grid.shape).T datasets = ParamStorage() datasets.data = ParamStorage() datasets.name = ParamStorage() datasets.name.fmt = fmt grid = grid.T datasets.data[name] = np.zeros([grid.shape[0],len(np.atleast_1d(names))])\ .astype(object) datasets.data[name][:, :] = None for i in xrange(len(np.atleast_1d(this_name))): ww = np.where(names == this_name[i])[0][0] datasets.data[name][:, ww] = grid[:, i] datasets.data.location = np.array(locations).T datasets.data.control = np.array(controls).T datasets.data.qlocation = np.array(qlocations).T datasets.name[name] = np.array(names) datasets.name.location = np.array(['time', 'row', 'col']) datasets.name.control = np.array(control) datasets.name.qlocation = limits datasets.name.bands = np.array(bands) datasets.data.sd = np.zeros([grid.shape[0],len(np.atleast_1d(names))])\ .astype(object) # for i in xrange(grid.shape[0]): # datasets.data.sd[i,:] = self.options.sd datasets.data.sd[:, :] = None for i in xrange(len(np.atleast_1d(this_name))): ww = np.where(names == this_name[i])[0][0] datasets.data.sd[:, ww] = sd[:, i] datasets.name.sd = np.array(names) if 'logger' in self or 'logger' in self.dict(): self.logger.debug('finished parsing dataset') except: self.error_msg=\ "a problem processing information from %s as a NpzFile"\ %filename self.error = True if 'logger' in self or 'logger' in self.dict(): self.logger.info(self.error_msg) return 0, (self.error, self.error_msg) f.close() if 'logger' in self or 'logger' in self.dict(): self.logger.info('... done') self.error = False self.error_msg = "" return datasets, (self.error, self.error_msg)
def postload_prepare(self): ''' This is called on initialisation, after data have been read in Here, we load parameters specifically associated with the model H(x). In the case of this differential operator, there are: model_order : order of the differential operator (integer) wraparound : edge conditions Can be: periodic none reflexive lag : The (time/space) lag at which the finite difference is calculated in the differential operator here. If this is 1, then we take the difference between each sample point and its neighbour. This is what we normally use. The main purpose of this mechanism is to allow differences at multiple lags to be calculated (fuller autocorrelation function constraints as in kriging) Multiple lags can be specified (which you could use to perform kriging), in which case lag weight should also be specified. lag_weight : The weight associated with each lag. This will generally be decreasing with increasing lag for a 'usual' autocorrelation function. There is no point specifying this if only a single lag is specified as the function is normalised. If the conditions are specified as periodic the period of the function can also be specified, e.g. for time varying data, you could specify 365 for the periodic period. These are specified in the configuration file as operator.modelt.rt_model.model_order operator.modelt.rt_model.wraparound operator.modelt.rt_model.lag operator.modelt.rt_model.lag_weight The default values (set here) are 1, 'none', 1 and 1 respectively. To specify the period for `periodic` specify e.g.: [operator.modelt.rt_model] wraparound=periodic,365 The default period is set to 0, which implies that it is periodic on whatever the data extent is. Or for multiple lags: [operator.modelt.rt_model] lag=1,2,3,4,5 lag_weight=1,0.7,0.5,0.35,0.2 NB this lag mechanism has not yet been fully tested and should be used with caution. It is intended more as a placeholder for future developments. Finally, we can also decide to work with inverse gamma (i.e. an uncertainty-based measure) This is achieved by setting the flag operator.modelt.rt_model.inverse_gamma=True This flag should be set if you intend to estimate gamma in the Data Assimilation. Again, the is experimental and should be used with caution. ''' from eoldas_Lib import sortopt self.rt_model = sortopt(self.options, 'rt_model', ParamStorage()) self.rt_model.lag = sortopt(self.rt_model, 'lag', 1) self.rt_model.inverse_gamma= \ sortopt(self.rt_model,'inverse_gamma',False) self.rt_model.model_order = \ sortopt(self.rt_model,'model_order',1) self.rt_model.wraparound = \ sortopt(self.rt_model,'wraparound','none') self.rt_model.wraparound_mod = 0 if np.array(self.rt_model.wraparound).size == 2 and \ np.array(self.rt_model.wraparound)[0] == 'periodic': self.rt_model.wraparound_mod = \ np.array(self.rt_model.wraparound)[1] self.rt_model.wraparound = \ np.array(self.rt_model.wraparound)[0] self.rt_model.lag = \ sortopt(self.rt_model,'lag',[1]) self.rt_model.lag = np.array(self.rt_model.lag).flatten() self.rt_model.lag_weight = \ sortopt(self.rt_model,'lag_weight',[1.]*\ self.rt_model.lag.size) self.rt_model.lag_weight = np.array(\ self.rt_model.lag_weight).flatten().astype(float) if self.rt_model.lag_weight.sum() == 0: self.rt_model.lag_weight[:] = np.ones( self.rt_model.lag_weight.size) self.rt_model.lag_weight = self.rt_model.lag_weight\ / self.rt_model.lag_weight.sum()
def J_prime_prime(self): ''' Calculation of J'' We already have the differntial operator self.linear.D1 and self.gamma after we call self.J_prime() Here, J'' = D1.T gamma^2 D1 J' is of shape (nobs,nstates) which is the same as the shape of x D1 is of shape (nobs,nobs) which needs to be expanded to (nobs,nstates,nobs,nstates) ''' x, Cx1, xshape, y, Cy1, yshape = self.getxy() J, J_prime = self.J_prime() xshape = self.x.state.shape if not 'linear' in self.dict(): self.linear = ParamStorage() if not 'J_prime_prime' in self.linear.dict(): self.linear.J_prime_prime = \ np.zeros(xshape*2) else: self.linear.J_prime_prime[:] = 0 # we need an indexing system in case of multiple # nobs columns x2a = np.diag(np.ones(self.linear.x2shape[:-1]).flatten()) try: gamma = self.linear.gamma.flatten() except: if self.linear.gamma_col != None: gamma = x.reshape(self.x.state.shape)\ [...,self.linear.gamma_col].flatten() else: # no gamma variable, so use 1.0 gamma = x.reshape(self.x.state.shape)\ [...,0].flatten()*0.+1. gamma = self.linear.gamma.flatten() if self.rt_model.inverse_gamma: tgamma = 1. / gamma dg = 2. / (gamma * gamma * gamma) else: tgamma = gamma dg = 1.0 nshape = tuple([np.array(self.linear.x2shape[:-1]).prod()]) D1 = np.matrix(self.linear.D1.reshape(nshape * 2)) i = 0 # so, e.g. we have xshape as (50, 100, 2) # because one of those columns refers to the gamma value # self.linear.gamma_col will typically be 0 for count in xrange(xshape[-1]): if count != self.linear.gamma_col: # we only want to process the non gamma col C1 = np.diag(self.linear.C1[...,i].\ reshape(self.linear.D1.shape[0])) xg = np.matrix(x2a * tgamma * tgamma) dxg = D1 * xg deriv = np.array(dxg.T * C1 * D1) # so we have gamma^2 D^2 which is the Hessian # we just have to put it in the right place now # the technical issue is indexing an array of eg # (50, 100, 2, 50, 100, 2) # but it might have more or fewer dimensions nd = len(np.array(xshape)[:-1]) nshape = tuple(np.array(xshape)[:-1]) if nd == 1: self.linear.J_prime_prime[:, count, :, count] = deriv.reshape(nshape * 2) elif nd == 2: self.linear.J_prime_prime[:, :, count, :, :, count] = deriv.reshape(nshape * 2) elif nd == 3: self.linear.J_prime_prime[:, :, :, count, :, :, :, count] = deriv.reshape(nshape * 2) else: self.logger.error( "Can't calculate Hessian for %d dimensions ... I can only do up to 3" % nd) #ww = np.where(deriv) #ww2 = tuple([ww[0]]) + tuple([ww[0]*0+count]) \ # + tuple([ww[1]] )+ tuple([ww[0]*0+count]) #x1 = deriv.shape[0] #x2 = self.linear.J_prime_prime.shape[-1] #xx = self.linear.J_prime_prime.copy() #xx = xx.reshape(x1,x2,x1,x2) #xx[ww2] = deriv[ww] #self.linear.J_prime_prime = xx.reshape(self.linear.J_prime_prime.shape) i += 1 if self.linear.gamma_col != None: c = self.linear.gamma_col nd = len(np.array(xshape)[:-1]) nshape = tuple(np.array(xshape)[:-1]) deriv = np.diag(dg * 2 * J / (tgamma * tgamma)).reshape(nshape * 2) if nd == 1: self.linear.J_prime_prime[:, c, :, c] = deriv elif nd == 2: self.linear.J_prime_prime[:, :, c, :, :, c] = deriv elif nd == 3: self.linear.J_prime_prime[:, :, :, c, :, :, :, c] = deriv else: self.logger.error( "Can't calculate Hessian for %d dimensions ... I can only do up to 3" % nd) #dd = np.arange(nshape[0]) #x1 = dd.shape[0] #x2 = self.linear.J_prime_prime.shape[-1] #xx = self.linear.J_prime_prime.copy() #xx = xx.reshape(x1,x2,x1,x2) #xx[dd,dd*0+self.linear.gamma_col,\ # dd,dd*0+self.linear.gamma_col] = dg*2*J/(tgamma*tgamma) #self.linear.J_prime_prime = xx.reshape(self.linear.J_prime_prime.shape) n = np.array(xshape).prod() return J, J_prime, self.linear.J_prime_prime.reshape(n, n)
def __init__(self, confs, logger=None, logfile=None, thisname=None, name=None, datadir=None, logdir=None): ''' Initialise the solver. This does the following: 1. Read configuration file(s) 2. Load operators 3. Test the call to the cost function There can be multiple groups of configuration files, so self.confs, that holds the core information setup here can contain multiple configurations. The number of configurations is len(confs.infos) and the ith configuration is conf = self.confs.infos[i]. Various loggers are available throughout the classes used, but the top level logger is self.confs.logger, so you can log with e.g. self.confs.logger.info('this is some info') The root operator is stored in self.confs.root[i] for the ith configuration, so the basic call to the cost function is: J,J_prime = self.confs[i].parameter.cost(None) ''' from eoldas_ConfFile import ConfFile from eoldas_Lib import sortopt name = name or thisname if name == None: import time thistime = str(time.time()) name = type(self).__name__ name = "%s.%s" % (name, thistime) self.thisname = name self.confs = confs self.top = sortopt(self, 'top', ParamStorage()) self.top.general = sortopt(self.top, 'general', ParamStorage()) thisname = sortopt(self.top.general, 'name', thisname or self.thisname) logfile = sortopt(self.top.general, 'logfile', logfile or 'log.dat') logdir = sortopt(self.top.general, 'logdir', logfile or 'logs') datadir = sortopt(self.top.general, 'datadir', datadir or ['.']) self.logger = sortlog(self,logfile,logger or self.confs.logger,\ name=self.thisname,logdir=logdir,debug=True) n_configs = len(self.confs.infos) self.confs.root = [] self.have_unc = False try: logdir = logdir except: logdir = self.top.general.logdir # first set up parameter conf = confs.infos[0] general = conf.general op = conf.parameter if not 'parameter' in conf.dict(): raise Exception('No parameter field found in %s item %d'%\ (conf.__doc__,0)) general.is_spectral = sortopt(general, 'is_spectral', True) if not general.is_spectral: sort_non_spectral_model(op, conf.operator, logger=confs.logger) general.init_test = sortopt(general, 'init_test', False) confs.logger.info('loading parameter state') conf.parameter.name = 'Operator' parameter = eval(op.name)(op,general,\ parameter=None,\ logger=confs.logger,\ name=name+".parameter",\ datatype=list(conf.parameter.datatypes),\ logdir=logdir,\ logfile=logfile,\ datadir=datadir) try: parameter.transform = parameter.options.x.transform parameter.invtransform = parameter.options.x.invtransform except: parameter.transform = parameter.options.x.names parameter.invtransform = parameter.options.x.names # we now have access to parameter.x.state, parameter.x.sd etc # and possibly parameter.y.state etc. operators = [] for (opname, op) in conf.operator.dict().iteritems(): if opname != 'helper': #pdb.set_trace() exec('from eoldas_%s import %s' % (op.name, op.name)) # make sure the data limits and x bounds are the same # ... inherit from parameter op.limits = parameter.options.limits if not 'datatypes' in op.dict(): op.datatypes = 'x' thisop = eval(op.name)(op,general,parameter=parameter,\ logger=confs.logger,\ name=name+".%s-%s"%(thisname,opname),\ datatype=list(op.datatypes),\ logdir=logdir,\ logfile=logfile,\ datadir=datadir) # load from parameter thisop.loader(parameter) operators.append(thisop) try: thisop.transform = parameter.options.x.transform thisop.invtransform = parameter.options.x.invtransform except: thisop.transform = parameter.options.x.names thisop.invtransform = parameter.options.x.names thisop.ploaderMask = np.in1d(parameter.x_meta.state, thisop.x_meta.state) try: thisop.invtransform = np.array( thisop.invtransform[thisop.ploaderMask]) thisop.transform = np.array( thisop.transform[thisop.ploaderMask]) except: ww = thisop.ploaderMask thisop.invtransform = np.array(thisop.transform)[ww] thisop.transform = np.array(thisop.transform)[ww] # sort the loaders parameter.operators = operators self.confs.root.append(parameter) # Now we have set up the operators # try out the cost function if general.init_test: self.logger.info('testing cost function calls') J, J_prime = self.confs.root[0].cost() self.logger.info('done') self.confs.root = np.array(self.confs.root)
def __init__(self, args, name=None, general=None, log=False, logger=None, outdir=".", getopdir=False, parse=True): """ Initialise parser class. This sets up the class defaults. Options: general=general: this over-rides and defaults with values set in parser general can be of the form: 1. class ParamStorage (i.e. the same form as self.general) 2. a command line list (where the first item in the list is ignored 3. a string containing a set of command line general See self.parse() for more details on general 2 and 3 as these simply make a call to tha method. log=True If log is set to True, then logging starts when this class is instanced. Note that the logfile and logdir might change if subsequent calls to Parser.parse() are made """ if type(args) == str: args = args.split() self.dolog = log self.log = log self.name = args[0] self.args = args[1:] self.fullargs = args self.store_fullargs = args if name == None: import time thistime = str(time.time()) name = type(self).__name__ name = "%s.%s" % (name, thistime) self.thisname = name # find the following flags: # --conf | -c : conf # --datadir : datadir datadir = [".","~/.eoldas",sys.path[0]+'/../bin',sys.path[0]+'/../confs',\ sys.path[0]+'/../system_confs',sys.path[0]+'/../eoldaslib'] conf = "default.conf" logfile = None logdir = "." self.top = ParamStorage() self.top.general = ParamStorage() self.top.general.__helper__ = ParamStorage() self.top.general.__default__ = ParamStorage() self.top.general.__extras__ = ParamStorage() self.top.general.conf = [] for i in xrange(len(self.args)): theseargs = self.args[i].split('=') if theseargs[0] == "--conf": conf = theseargs[1] self.top.general.conf.append(conf) elif theseargs[0][0:2] == "-c": if len(theseargs) > 2: conf = theseargs[0][2:] else: conf = self.args[i + 1] self.top.general.conf.append(conf) elif theseargs[0] == "--datadir": datadir1 = theseargs[1].replace('[','').\ replace(']','').split() [datadir1.append(datadir[i]) for i in \ xrange(len(datadir))] datadir = datadir1 elif theseargs[0] == "--logfile": logfile = theseargs[1] elif theseargs[0] == "--logdir": logdir = theseargs[1] elif theseargs[0] == "--outdir": outdir = theseargs[1] if self.top.general.conf == []: self.top.general.conf = conf if logfile == None: logfile = conf.replace('conf', 'log') self.top.general.here = os.getcwd() self.top.general.datadir = datadir self.top.general.logfile = logfile self.top.general.logdir = logdir self.top.general.outdir = outdir # add here to datadir # in addition to '.' to take account of the change of directory self.top.general.datadir = self.__add_here_to_datadir(\ self.top.general.here,self.top.general.datadir) self.top.general.datadir = self.__add_here_to_datadir(\ self.top.general.outdir,self.top.general.datadir) # cd to where the output is to be self.__cd(self.top.general.outdir) # set up the default command line options self.default_loader() # update with anything passed here if general and type(general) == ParamStorage: self.top.update(\ self.__unload(general),combine=True) # read the conf files to get any cmd line options self.logger = sortlog(self,self.top.general.logfile,logger,name=self.thisname,\ logdir=self.top.general.logdir) self.config = ConfFile(self.top.general.conf,name=self.thisname+'.config',\ loaders=self.loaders,datadir=self.top.\ general.datadir,logger=self.logger,logdir=self.top.general.logdir,\ logfile=self.top.general.logfile) if len(self.config.configs) == 0: this = "Warning: Nothing doing ... you haven't set any configuration",\ self.config.storelog try: self.logger(this) except: print "Called with args:" print "eoldas", self.args pass raise Exception(this) # now loaders contains all of the defaults set here # plus those from the config (config opver-rides defaults here) self.loaders = self.config.loaders # now convert loaders into parser information self.parseLoader(self.loaders) self.parse(self.fullargs) if general and type(general) == ParamStorage: self.top.update(self.__unload(general), combine=True) if general and type(general) == str: self.parse(general.split()) if general and type(general) == list: self.parse(general) # now update the info in self.config for i in self.config.infos: i.update(self.top, combine=True) # so now all terms in self.config.infos # contain information from the config file, updated by # the cmd line i.logger = self.logger i.log() # move the information up a level self.infos = self.config.infos self.configs = self.config.configs self.config_log = self.config.storelog #if getopdir: # self.sortnames() self.config.loglist(self.top) #del(self.config.infos) #del(self.config.configs) #del(self.config) self.__cd(self.top.general.here)
def __setattr__(self,this,value): ''' Variable setting method for style self.this Varies what it does depending on the type of value. The method interprets and sets the SpecialVariable value: 1. ParamStorage or SpecialVariable. The data are directly loaded. This is one of the most flexible formats for input. It expects fields 'data' and/or 'name', which are loaded into self. There will normally be a field data.this, where this is the variable name passed here. 2. A dictionary, same format as the ParamStorage. 3. A tuple, interpreted as (data,name) and loaded accordingly. 4. *string* as filename (various formats). An attempt to read the string as a file (of a set of formats) is made. If none pass then it it maintained as a string. 5. A numpy array (np.array) that is loaded into self.data.this. 6. Anything else. Loaded into self.data.this as a numpy array. ''' if self.simple: self.set(this,value) return t = type(value) try: if t == ParamStorage or t == SpecialVariable: # update the whole structure #self.__set_if_unset('data',ParamStorage()) #self.__set_if_unset('name',ParamStorage()) self.data.update(value.data,combine=True) self.name.update(value.name,combine=True) elif t == dict: n_value = ParamStorage().from_dict(value) self.__setattr__(this,n_value) elif t == tuple or t == list: # assumed to be (data,name) or [data,name] #self.__set_if_unset('data',ParamStorage()) #self.__set_if_unset('name',ParamStorage()) #ParamStorage.__setattr__(self['data'],this,value[0]) #ParamStorage.__setattr__(self['name'],this,value[1]) ParamStorage.__setattr__(self['data'],this,np.array(value)) elif t == str: # set the term #self.__set_if_unset('data',ParamStorage()) #self.__set_if_unset('name',ParamStorage()) ParamStorage.__setattr__(self['data'],this,value) # interpret as a file read if possible self.process_data_string(this,info=self.info) elif t == np.ndarray: #self.__set_if_unset('data',ParamStorage()) #self.__set_if_unset('name',ParamStorage()) ParamStorage.__setattr__(self['data'],this,value) else: ParamStorage.__setattr__(self['data'],this,\ np.array(value)) except: if self.logger: self.logger.info("Failed to set SpecialVariable %s from %s %s"\ %(this,t.__name__,value)) return if self.logger: self.logger.info("Set variable %s from type %s"%(this,t.__name__))
def prep(self, thisconf): ''' A method to prepare the solver ''' self.root = self.confs.root[thisconf] root = self.confs.root[thisconf] self.sortMask() self.op = sortopt(root.general, 'optimisation', ParamStorage()) self.op.plot = sortopt(self.op, 'plot', 0) self.op.name = sortopt(self.op, 'name', 'solver') self.op.maxfunevals = sortopt(self.op, 'maxfunevals', 2e4) self.op.maxiter = sortopt(self.op, 'maxiter', 1e4) self.op.gtol = sortopt(self.op, 'gtol', 1e-3) self.op.iprint = sortopt(self.op, 'iprint', 1) self.op.solverfn = sortopt(self.op, 'solverfn', 'scipy_lbfgsb') self.op.randomise = sortopt(self.op, 'randomise', False) self.op.no_df = sortopt(self.op, 'no_df', False) self.result = sortopt(root.options, 'result', ParamStorage()) self.result.filename = sortopt(root.options.result,'filename',\ 'results.pkl') self.result.fmt = sortopt(root.options.result, 'format', 'pickle') try: self.transform = self.root.options.x.transform self.invtransform = self.root.options.x.transform except: self.transform = None self.invtransform = None # descend into the operators and identify any observation ops # we then want to be able to write out files (or plot data) # that are H(x). # We only do this for Operators that have both 'x' and 'y' # terms as we store the filename under 'y.result' self.Hx_ops = [] for i, op in enumerate(root.operators): op.loader(root) if 'y_meta' in op.dict() and op.options.y.datatype == 'y': # There is a potential observation op.y_state.options.y.result = \ sortopt(op.y_state.options.y,'result',ParamStorage()) op.y_state.options.y.result.filename = \ sortopt(op.y_state.options.y.result,\ 'filename',op.y_state.thisname) op.y_state.options.y.result.format = \ sortopt(op.y_state.options.y.result,\ 'format','PARAMETERS') state = op.y_state._state this = { \ 'filename':op.y_state.options.y.result.filename,\ 'format':op.y_state.options.y.result.format,\ 'state':state,\ 'y':op.y_state,\ 'op':op,\ 'transform':self.transform,\ 'invtransform':self.invtransform,\ 'Hx':op.linear.H} op.Hx_info = this self.Hx_ops.append(this) else: this = { \ 'transform':self.transform,\ 'invtransform':self.invtransform,\ } op.Hx_info = this
def _load_rt_library(self, rt_library): """ A method that loads up the compiled RT library code and sets some configuration options. This method tries to import all the methods and make them available through `self.rt_library.<method_name>`. It is also a safe importer: if some functions are not available in the library, it will provide safe methods from them. Additionally, while importing, a number of configuration options in the class are also updated or set to default values. Parameters ----------- rt_library : string This is the name of the library object (.so file) that will be loaded """ from eoldas_Lib import sortopt import_string = "from %s import " % (rt_library) self.logger.debug("Using %s..." % rt_library) self.rt_library = sortopt(self, 'rt_library', ParamStorage()) # 1. Import main functionality try: self.logger.debug("Loading rt_model") exec(import_string + "rt_model") self.rt_library.rt_model = rt_model except ImportError: self.logger.info(\ "Could not import basic RT functionality: rt_model") self.logger.info(\ "Check library paths, and whether %s.so is available" % \ rt_library) raise Exception('error importing library %s' % rt_library) # 1a. Import conditioning methods that can be ignored try: exec(import_string + 'rt_modelpre') self.rt_library.rt_modelpre = rt_modelpre except: self.rt_library.rt_modelpre = self._nowt try: exec(import_string + 'rt_modelpre') self.rt_library.rt_modelpost = rt_modelpre except: self.rt_library.rt_modelpost = self._nowt # 2. Try to import derivative self.rt_model.ignore_derivative = sortopt(self.rt_model,\ 'ignore_derivative',False) if self.rt_model.ignore_derivative == False: try: exec ( import_string + \ "rt_modeld, rt_modeldpre, rt_modeldpost" + \ ", rt_modelpred" ) self.rt_model.have_adjoint = True self.J_prime = self.J_prime_full self.rt_library.rt_modeld = rt_modeld self.rt_library.rt_modeldpre = rt_modeldpre self.rt_library.rt_modeldpost = rt_modeldpost self.rt_library.rt_modelpred = rt_modelpred except ImportError: self.logger.info( "No adjoint. Using finite differences approximation.") self.rt_model.have_adjoint = False else: self.logger.info( "ignoring adjoint. Using finite differences approximation.") self.rt_model.have_adjoint = False self._configure_adjoint() try: exec(import_string + "rt_model_deriv") self.rt_library.rt_model_deriv = rt_model_deriv except ImportError: self.rt_library.rt_model_deriv = None
def init(self,info=[],name=None,readers=[],log_terms={},\ datadir=None,env=None,\ header=None,writers={},\ simple=False,logger=None): ''' Initialise information in a SpecialVariable instance. Options: info : Information tthat can be passed through to reader methods (a list). thisname : a name to use to identify this instance in any logging. By default this is None. If thisname is set to True, then logging is to stdout. readers : A list of reader methods that are pre-pended to those already contained in the class. log_terms : A dictionary of log options. By default {'logfile':None,'logdir':'log','debug':True} If thisname is set, and logfile specified, then logs are logged to that file. If thisname is set to True, then logging is to stdout. datadir : A list of directories to search for data files to interpret if the SpecialVariable is set to a string. env : An environment variable that can be used to extend the datadir variable. header : A header string to use to identify pickle files. By default, this is set to "EOLDAS -- plewis -- UCL -- V0.1" simple : A flag to swicth off the 'complicated' interpretation methods, i.e. just set and return variables literally, do not try to interpret them. ''' self.set('simple',True) if name == None: import time thistime = str(time.time()) name = type(self).__name__ name = "%s.%s" % (name,thistime) self.thisname = name # this is where we will put any data self.data = ParamStorage() self.name = ParamStorage() self.info = info self.datadir = datadir or ['.'] self.env = env init_read_write(self,header,readers,writers) # sort logging and log if thisname != None self.log_terms = {'logfile':None,'logdir':'log','debug':True} # override logging info for (key,value) in log_terms.iteritems(): self.log_terms[key] = value self.logger= sortlog(self,self.log_terms['logfile'],logger,name=self.thisname,\ logdir=self.log_terms['logdir'],\ debug=self.log_terms['debug']) self.simple = simple
def read_single_conf_file(self, conf_files, options=None): """ Purpose: parse the information from conf_files into a ConfigParser class instance and return this. Parameters: conf_files : list of one or more config files Options: options=None : pass an options structure through Uses: self.datadir=['.',,'~/.eoldas'] : list of directories to look for config files self.env=None : name of an environment variable where config files can be searched for if not found in datadir (or absolute path name not given) self.fatal=False : flag to state whether the call should fail if a requested config file is not found. Returns: tuple : (config, config_error) where: config : ConfigParser class instance or False if an error occurs config_error : string giving information on error """ import ConfigParser from eoldas_Lib import get_filename # Instantiate a parser config = ConfigParser.ConfigParser() # Read the config files. If it doesn't exist, raise exception. # if type(conf_files) == str: conf_files = [conf_files] all_conf_files = [] for fname in conf_files: fname,fname_err = get_filename(fname,datadir=self.datadir,\ env=self.env) if fname_err[0] != 0: if self.fatal: return False,False,\ "Cannot find configuration file %s\n%s" \ % (fname,fname_err[1]) else: all_conf_files.append(fname) thisdir = os.path.dirname(fname) if not thisdir in self.datadir: self.datadir.append(thisdir) if len(all_conf_files) == 0: return False,False,\ "%s: No valid conf files found in list %s in dirs %s" \ % (os.getcwd(),conf_files,self.datadir) config.config_files = config.read(all_conf_files) if len(config.config_files) == 0: return False,False,\ "%s: No valid conf files found in list %s in dirs %s" \ % (os.getcwd(),conf_files,self.datadir) # from here on, we attempt to pull specific information from # the conf files info = ParamStorage(name='info',doc=\ 'Configuration information for %s' % \ str(config.config_files)) # scan everything into config.info # but it helps to sort it to get the info in the right order sections = config.sections() #sections.sort() firstsections = [] secondsections = [] for this in sections: if this[:7] == 'general' or this[:9] == 'parameter': firstsections.append(this) else: secondsections.append(this) firstsections.sort() sections = firstsections [sections.append(i) for i in secondsections] for this in sections: self.logger.debug('...Section %s' % this) self.scan_info(config, this, info, this, info) self.rescan_info(config, this, info, this, info, 0) self.config = config self.info = info if options != None and type(options) == ParamStorage: self.info.update(options, combine=True) # sort any helper text looping over self.info # into self.loaders self.__sort_help(self.info, "") try: self.logger.info("Config: %s read correctly" \ % str(all_conf_files)) except: pass return self.config,self.info,"Config: %s read correctly" \ % str(all_conf_files)
def demonstration(): # set state to a filename # and it will be loaded with the data x = DemoClass() data = {'state':np.ones(2)*5. ,'foo':np.ones(10)} name = {'state':'of the nation','foo':'bar'} this = {'data':data,'name':name} x.state = this print 1,x.state,x['state'] x.oats = 'beans and barley-o' # nothing set so far print 2,x['state'] # should return the same print 3,x.state x.state = 'test/data_type/input/test.brf' print 4,x.state print 5,x.Name.fmt # set state to a dict and # it will load from that data = {'state':np.zeros(10)} name = {'state':'foo'} x.state = {'data':data,'name':name} print 6,x.state # set from a ParamStorage # and it will be loaded this = ParamStorage() this.data = ParamStorage() this.name = ParamStorage() this.data.state = np.ones(10) this.name.state = 'bar' this.data.sd = np.ones(10)*2. this.name.sd = 'sd info' # assign the data x.state = this # access the data print 7,x.state # access another member # Data, Name == implicitly .state print 8,x.Data.sd print 9,x.Name.sd # set directly x.Name.sd = 'bar' print 10,x.Name.sd # set from a tuple (data,name) # or a list [data,name] data = 'foo' name = 'bar' x.state = (data,name) print 11,x.state x.state = [name,data] print 12,x.state # set from a numpy array x.state = np.array(np.arange(10)) print 13,x.state # set from another state y = DemoClass() y.state = x.state x.state = x.state * 2 print 'x state',x.state print 'y state',y.state # set from a float x.state = 100. print 14,x.state # another interesting feature # we have 2 special terms in demonstration # state and other # if we set up some strcture for data # for other this = ParamStorage() this.data = ParamStorage() this.name = ParamStorage() this.data.other = np.ones(10) this.name.other = 'bar' # and the assign it to state x.state = this print 15,'state',x.state # we see state is unchanged # but other is also not set. print 16,'other',x.other # we load into other using: x.other = this print 'other',x.other # but if you look at the information contained print 17,x._other.to_dict() print 18,x._state.to_dict() # or better writtem as: print 19,x.var('state').to_dict() # you will see that state contains the other data that was loaded # a simple way to write out the data is to a pickle # x.write_pickle('xstate','x_state.pkl') # but try to avoid using the underscores print 20,"x state in pickle:",x.state SpecialVariable.write(x._state,'x_state.pkl',fmt='pickle') # which we can reload: z = DemoClass() z.state = 'x_state.pkl' print 21,"z state read from pickle",z.state # which is the same as a forced read ... zz = DemoClass() zz.state = 'x_state.pkl' print 22,zz.state # read a brf file zz.Name.qlocation = [[170,365,1],[0,500,1],[200,200,1]] zz.state = 'test/data_type/input/interpolated_data.npz' print zz.state SpecialVariable.write(zz._state,'test/data_type/output/interpolated_data.pkl',fmt='pickle') zz.state = 'test/data_type/input/test.brf' print zz.state
def regrid(self): ''' Utility to regrid non-gridded state (& associated) data If no data are specified (we can see this because self.Data.qlocation doesnt exist) then a default grid is generated. Outputs: self.Name.gridder self.Data.gridder containing: grid : state vector grid (offset in name) sdgrid : state vector sd grid (offset in name) ngrid : n samples from ip data for grids wheregrid : where grid points are data points ''' try: qlocation = self.Data.qlocation has_data = True self.logger.info("Looking at loading a grid") except: nd = len(self.Name.qlocation) qlocation = np.zeros((2, nd)) for i in xrange(nd): qlocation[:, i] = self.Name.qlocation[0][0:2] # no data defined, so set up a default grid has_data = False self.logger.info("No input data given: using default grid") if type(self.Data.state) == str: # somehow the magic reader hasnt worked raise Exception("Datafile %s hasn't been interpreted as state data"\ %self.Data.state) if np.array(self.Data.state).size == 1 and np.array( self.Data.state) == np.array(None): has_data = False datatype = self.options.datatype if datatype == 'y': self.logger.info("Not loading grid as datatype is y") return try: if not self.options[datatype].apply_grid: self.logger.info("Not loading grid as apply_grid not set") return except: return try: default = self.options[datatype].default except: default = np.zeros_like(self.Name.state).astype(float) try: sddefault = self.options[datatype].sddefault except: sddefault = list(np.array(default) * 0.) self.ungridder = ParamStorage() nd = qlocation.shape[1] limits = self.Name.qlocation nparams = len(default) x = [] minx = [] stepx = [] # number of location dimensions == nd for i in xrange(nd): xmin = qlocation[:, i].min() xmax = qlocation[:, i].max() xstep = 1 lim = limits[i] if lim[0] != None: xmin = lim[0] if lim[1] != None: xmax = lim[1] if lim[2] != None: xstep = lim[2] # LEWIS: 20 June 2012 ERROR if xstep used # x.append((xmax-xmin + 1)/xstep) x.append((xmax - xmin) / xstep + 1) minx.append(xmin) stepx.append(xstep) self.Name.qlocation_min = minx self.Name.qlocation_step = stepx # x contains the number of desired samples # in each dimension x.append(nparams) minx.append(0) stepx.append(0) # now loop over all observations and place in grid ntot = np.array(x).prod() grid = np.zeros(ntot).reshape(tuple(x)) sdgrid = np.zeros(ntot).reshape(tuple(x)) ngrid = np.zeros(ntot / x[-1], dtype=int).reshape(tuple(x[:nd])) # now fill the grid all = ':,' for i in xrange(1, nd): all = '%s:,' % all for i in xrange(nparams): if i >= len(default): self.logger.error("Incorrect length for default") if i >= len(sddefault): self.logger.error("Incorrect length for sd") exec('grid[%s%d] = default[%d]' % (all, i, i)) exec('sdgrid[%s%d] = sddefault[%d]' % (all, i, i)) if has_data: for i in xrange(qlocation.shape[0]): loc = tuple(qlocation[i, :]) #-minx[:nd]) ngrid[loc] += 1 thisdata = self.Data.state[i] if ngrid[loc] == 1: grid[loc][:] = thisdata else: grid[loc][:] += thisdata # LEWIS 20 June 2012 if self.Name.datatype == 'x': # take mean for loc in np.where(ngrid > 1)[0]: grid[loc][:] = grid[loc][:] / float(ngrid[loc]) wheregrid = np.where(ngrid > 0) self.Name.gridder = ParamStorage() self.Name.gridder.nd = len(self.Name.qlocation) self.Data.gridder = ParamStorage() self.Data.gridder.grid = grid self.Data.gridder.ngrid = ngrid self.Data.gridder.sdgrid = sdgrid self.Data.gridder.wheregrid = wheregrid self.Name.gridder.grid = minx self.Name.gridder.sdgrid = minx self.Name.gridder.stepx = stepx self.Name.gridder.wheregrid = wheregrid[0].size self.Data.gridder.ngrid = x # now form the information needed for ungridding state = grid nloc = len(state.shape) - 1 ss = np.array(state.shape) ss[-1] = nloc ss = tuple(ss) qlocation_min = minx qlocation_step = stepx locations = np.zeros(ss) qlocations = np.zeros(ss, dtype=int) for i in xrange(nloc): aa = np.zeros(nloc, dtype=object) for jj in xrange(ss[i]): if i == 0: qlocations[jj, ..., i] = jj locations[jj, ..., i] = jj * stepx[i] + qlocation_min[i] elif i == 1: qlocations[:, jj, ..., i] = jj locations[:, jj, ..., i] = jj * stepx[i] + qlocation_min[i] elif i == 2: qlocations[:, :, jj, ..., i] = jj locations[:, :, jj, ..., i] = jj * stepx[i] + qlocation_min[i] elif i == 3: qlocations[:, :, :, jj, ..., i] = jj locations[:, :, :, jj, ..., i] = jj * stepx[i] + qlocation_min[i] elif i == 4: qlocations[:, :, :, :, jj, ..., i] = jj locations[:, :, :, :, jj, ..., i] = jj * stepx[i] + qlocation_min[i] else: raise Exception('How many dimensions in your dataset ??? > 4 ??? thats ridiculous'+\ " ... I can't write that ") self.ungridder.qlocation = qlocations self.ungridder.location = locations self.ungridder.nloc = nloc