Exemplo n.º 1
0
    def __init__(self,info=[],name=None,thisname=None,readers=[],log_terms={},\
                                                datadir=["."],env=None,\
                                                header=None,writers={},
                                                simple=False,logger=None):
        '''
        Class SpecialVariable initialisation.
            
        Sets up the class as a ParamStorage and calls self.init()
            
        See init() fort a fuller descripotion of the options.
            
        '''
        ParamStorage.__init__(self,logger=logger)
        name = name or thisname
        if name == None:
           import time
           thistime = str(time.time())
           name = type(self).__name__
           name =  "%s.%s" % (name,thistime)
        self.thisname = name

        self.init(info=[],name=self.thisname,readers=readers,log_terms={},\
                                                datadir=datadir,env=env,\
                                                header=header,writers=writers,\
                                                simple=False,logger=logger)
Exemplo n.º 2
0
    def __setitem__(self,name,value,nocheck=False):
        '''
            set item for class e.g. self['state'] = 3
            '''
        if nocheck or not self.__set_if_unset(name,value):
            if name in ['Data','Name']:
                self._state[name.lower()] = value
            elif name in ['Control','Location']:
                self._state[name.lower()] = value
            elif name == 'state':
                # NB 'self' during the __setitem__ call
                # will be self._state as far as we are 
                # concerned here
                try:
                    self._state.name.datatype = self.datatype
                except:
                    pass
                SpecialVariable.__setattr__(self._state,name,value)
                #super( State, self ).__setattr__(name,value)
                # apply any overrides from options
                self.apply_defaults(self._state,self.options)
                if self.grid:
                    if 'location' in self.Name.dict().keys():
			#pdb.set_trace()
                        self.regrid()
            else:
                ParamStorage.__setattr__(self,name,value)
Exemplo n.º 3
0
 def __setitem__(self, name, value, nocheck=False):
     '''
         set item for class e.g. self['state'] = 3
         '''
     if nocheck or not self.__set_if_unset(name, value):
         if name in ['Data', 'Name']:
             self._state[name.lower()] = value
         elif name in ['Control', 'Location']:
             self._state[name.lower()] = value
         elif name == 'state':
             # NB 'self' during the __setitem__ call
             # will be self._state as far as we are
             # concerned here
             try:
                 self._state.name.datatype = self.datatype
             except:
                 pass
             SpecialVariable.__setattr__(self._state, name, value)
             #super( State, self ).__setattr__(name,value)
             # apply any overrides from options
             self.apply_defaults(self._state, self.options)
             if self.grid:
                 if 'location' in self.Name.dict().keys():
                     #pdb.set_trace()
                     self.regrid()
         else:
             ParamStorage.__setattr__(self, name, value)
Exemplo n.º 4
0
def sortlog(self,logfile,logger,name="eoldas",logdir=None,debug=True ):
    '''
    A safe interface to logging
    for passing log information between lots of classes
    '''
    import logging
    import time
    from eoldas_Lib import set_up_logfile,dummyprint

    if type(self).__name__  == 'SpecialVariable' and logger != None:
        return logger
    try:
        if 'logger' in self.dict():
            return self.logger
    except:
        pass
    try:
        if 'logger' in self.keys():
            return self.logger
    except:
        pass
    try:
        if type(self).__name__ != 'SpecialVariable':
            this = self.logger
            return self.logger
    except:
        pass

    if logdir == None:
        logdir = '.'

    if name == None:
        name = type(self).__name__ + '.' + str(time.time())

    if logger:
        if type(self).__name__  == 'SpecialVariable':
            return logger
        logger.info('Setting up logger for %s'%name)
        logger = logging.getLogger(name)
	logger.info('Set up')
        return logger
        
    if logfile == None or name == None:
        logger = ParamStorage()
        logger.info  = lambda x:dummyprint( "Info: %s"%x)
        logger.debug = lambda x:dummyprint( "Debug: %s"%x)
        logger.error = lambda x:dummyprint( "Error: %s"%x)
        return logger
    logger = set_up_logfile(logfile,name=name,logdir=logdir,debug=debug)
    return logger
Exemplo n.º 5
0
 def __set_if_unset(self,name,value):
     '''
     A utility to check if the requested attribute
     is not currently set, and to set it if so.
     '''
     if name in self.fakes:
         fname = self.fakes[name]
         if not fname in self.__dict__:
             ParamStorage.__setattr__(self,fname,value)
             return True
     else:
         if not name in self.__dict__:  
             ParamStorage.__setattr__(self,name,value)
             return True
     return False
Exemplo n.º 6
0
 def __setitem__(self,name,value,nocheck=False):
     '''
         set item for class e.g. self['state'] = 3
     '''
     if nocheck or not self.__set_if_unset(name,value):
         if name in ['Data','Name']:
             self._state[name.lower()] = value
         elif name in ['Control','Location']:
             self._state[name.lower()] = value
         elif name in self.fakes:
             this = self.get(self.fakes[name])
             SpecialVariable.__setattr__(this,name,value)
         else:
             this = self.get(name)
             ParamStorage.__setattr__(self,name,value)
Exemplo n.º 7
0
    def __init__(self,argv,name='eoldas',logger=None):
        from eoldas.eoldas_Lib import sortopt, sortlog

        argv = argv or sys.argv
        here = os.getcwd()
        self.thisname = name
        Parser.__init__(self,argv,name=self.thisname,logger=logger,\
            general=None,outdir=".",getopdir=False,parse=True)
        os.chdir(here)
        if not hasattr(self,'configs'):
            self.logger.error('No configration file specfied')
            help(eoldas)
            return

        self.thisname = name
        solver = eoldas_Solver(self,logger=self.logger,name=self.thisname+'.solver')
        self.general = sortopt(self.root[0],'general',ParamStorage())
        self.general.write_results = sortopt(self.general,'write_results',True)
        self.general.calc_posterior_unc = sortopt(self.general,'calc_posterior_unc',False)
        self.general.passer = sortopt(self.general,'passer',False)
        self.solver = solver
        self.logger.info('testing full cost functions')
        for i in xrange(len(solver.confs.infos)):
            self.logger.info('%d/%d ...'%(i+1,len(solver.confs.infos)))
            # try an initial solver.prep(i)
            J = solver.cost(None)
            J_prime = solver.cost_df(None)
            self.logger.info('done')

        # give the user some info on where the log file is
        # in case theyve forgotten
        print 'logging to',self.general.logfile
Exemplo n.º 8
0
 def __unload(self,options):
     from eoldas_ConfFile import array_type_convert
     this = ParamStorage()
     this.general = ParamStorage()
     for (k,v) in options.iteritems():
         ps = this
         that = k.split('.')
         if len(that) == 1:
             ps = this.general
         else:
             for i in xrange(len(that)-1):
                 if not hasattr(ps,that[i]):
                     ps[that[i]] = ParamStorage()
                 ps = ps[that[i]]
         # set the value v which needs to 
         # to be interpreted
         ps[that[-1]] = array_type_convert(self.top,v)
     return this
Exemplo n.º 9
0
 def __unload(self, options):
     from eoldas_ConfFile import array_type_convert
     this = ParamStorage()
     this.general = ParamStorage()
     for (k, v) in options.iteritems():
         ps = this
         that = k.split('.')
         if len(that) == 1:
             ps = this.general
         else:
             for i in xrange(len(that) - 1):
                 if not hasattr(ps, that[i]):
                     ps[that[i]] = ParamStorage()
                 ps = ps[that[i]]
         # set the value v which needs to
         # to be interpreted
         ps[that[-1]] = array_type_convert(self.top, v)
     return this
Exemplo n.º 10
0
def read_numpy_fromfile(self,goodfile,dataset,info=None):
    '''
    Utility to try to read a file goodfile

    This simple version of the utility just tries a numpy.fromfile

    It is not generally recommended to use this first
    as it will skip headers etc that may contain
    information

    However, this is a good illustration of the required interface format
    for file readers.


    Inputs:
        goodfile  : a filename (ideally one that exists)
        dataset   : dataset name (e.g. x_state)
        info      : a list of other information

    Outputs:
        retval    : ParamStorage
        error     : tuple

    where:
         retval contains dataset 'this' in data['this']
         and other information (e.g. locations etc) in
         retval.data and retval.names

    '''
    #self.store_header = goodfile.open().readline().close()
    this = np.genfromtxt(goodfile,skip_header=1)
    l = len(np.atleast_1d(this).shape)
    if this.size > 0:
        retval = ParamStorage()
        retval.name = ParamStorage()
        retval.data = ParamStorage()
        retval.data[dataset] = this
        retval.name.filename = goodfile
        retval.name.fmt = 'np.genfromtxt'
        return retval,(False,"")
    error = True
    error_msg = "Failed numpy read of %s"%goodfile
    return 0,(error,error_msg)
Exemplo n.º 11
0
def safesplit(text, character):
    '''
    A function to split a string, taking account of [] and ()
    and quotes
    '''
    lent = len(text)
    intoken = ParamStorage()
    qtoken = ParamStorage()
    for i in ["'",'"']:
        qtoken[i] = False
    for i in ["()","[]","{}"]:
        intoken[i] = 0

    start = 0
    lst = []
    i = 0
    while i < lent:
        # are any of intoken, qtoken open
        isopen = False
        for (j,k) in intoken.iteritems():
            if text[i] == j[0]:
                intoken[j] += 1
            elif text[i] == j[1]:
                intoken[j] -= 1
            isopen = isopen or (intoken[j]!=0)
        for (j,k) in qtoken.iteritems():
            if text[i] == j[0]:
                qtoken[j] = not qtoken[j]
            isopen = isopen or qtoken[j]
        if text[i] == character and not isopen:
            lst.append(text[start:i])
            start = i+1
        elif text[i] == '\\':
            i += 2
            continue
        i += 1
    lst.append(text[start:])
    return lst
Exemplo n.º 12
0
 def __init__ ( self, nbands, nbands_max, npt, bandwith, obscovar, \
                     location, whichfile, doys, qa, theta_v, theta_i, \
                     phi_v, phi_i, isobs,  params_x, \
                     obs=0.0, bandpass_library=False ) :
                        
     # Define a configuration container. Makes everything look like Java
     # Containers can then be useful for quickly listing all variables :)
     self.config = ParamStorage ()
     self.config.spectral = ParamStorage ()
     self.config.rt_model = ParamStorage ()
     self.observations = ParamStorage ()
     
     self.config.rt_model.nparams = rt_getnparams ()
     
     self.config.npt = npt # Number of points
     self.config.nv = 1 # Always set to 1. No questions asked
     
     self._setup_spectral_config ( nbands, nbands_max, bandwith )
     self._setup_rt_model ()
     self._setup_bandpass_funcs ( bandwidth, bandpass_library )
     self._setup_geometry ( theta_v, phi_v, theta_i, phi_i )
     
     self.params_x = np.zeros (( self.config.npt, \
                     self.config.rt_model.nparams ) )
     self.observations.brf = np.zeros([self.config.npt, \
                             self.config.spectral.nbands_max] )
     self.observations.obs = np.zeros([self.config.npt, \
                             self.config.spectral.nbands_max] )
     # obs will be set to zero if not loaded
     self.observations.obs[:,:] = obs
     # setting self.brf_ad to 1 means that we calculate the model derivative 
     # by default
     self.brf_ad = np.ones([self.config.npt, \
                             self.config.spectral.nbands_max] )
     self._set_minmax ( )
     self._set_x( x )
Exemplo n.º 13
0
    def startlog(self, log_terms, name=None):
        '''
        Start the logger.

        This is called on initialisation and you shouldn't 
            normally need to access it.
        '''
        import logging
        from eoldas_Lib import set_up_logfile
        try:
            self.logger.shutdown()
        except:
            self.logger = ParamStorage()
        logfile = log_terms['logfile'] or self.options.logfile
        logdir = log_terms['logdir'] or self.options.logdir
        name = name or self.options.thisname
        self.logger = set_up_logfile(logfile,\
                                     name=name,logdir=logdir)
Exemplo n.º 14
0
    def setup_rt_model(self):
        """
        This sets up the RT model (and adjoint if available)
        by calling any preparation methods.
            
        """
        if not 'linear' in self.dict():
            self.linear = ParamStorage()
        if 'y' in self.dict():
            self.linear.H = np.zeros(self.y.state.shape)
        else:
            self.linear.H = np.zeros(self.x.state.shape)
        self.nv = 1
        self.npt = len(self.y.state)

        self.linear.J_prime = np.zeros(self.x.state.shape)

        if not self.rt_model.use_median:
            self.bandIndex = self.y_meta.spectral.all_bands
        else:
            self.bandIndex = self.y_meta.spectral.median_bands
        self.linear.brf_ad = np.ones((self.npt, len(self.bandIndex)))

        self.rt_library.rt_modelpre(np.array(self.bandIndex) + 1)
        self.rt_library.rt_modeldpre(self.npt)

        self.x_orig = self.x.state.copy()
        if self.rt_model.use_median:
            bands_to_use = self.y_meta.spectral.median_bands_to_use
            bandpass_library = self.y_meta.spectral.median_bandpass_library
            index = self.y_meta.spectral.median_bandpass_index
        else:
            bands_to_use = self.y_meta.spectral.bands_to_use
            bandpass_library = self.y_meta.spectral.bandpass_library
            index = self.y_meta.spectral.bandpass_index

        self.y_meta.spectral.bands_to_use = \
                    np.zeros((len(bands_to_use),len(self.bandIndex)))
        for (i, bandname) in enumerate(self.y_meta.spectral.bandnames):
            fullb = bandpass_library[bandname]
            this = fullb[index[bandname]]
            this = this / this.sum()
            ww = np.where(np.in1d(self.bandIndex, index[bandname]))[0]
            self.y_meta.spectral.bands_to_use[i, ww] = this
Exemplo n.º 15
0
def read_numpy_fromfile(self, goodfile, dataset, info=None):
    '''
    Utility to try to read a file goodfile

    This simple version of the utility just tries a numpy.fromfile

    It is not generally recommended to use this first
    as it will skip headers etc that may contain
    information

    However, this is a good illustration of the required interface format
    for file readers.


    Inputs:
        goodfile  : a filename (ideally one that exists)
        dataset   : dataset name (e.g. x_state)
        info      : a list of other information

    Outputs:
        retval    : ParamStorage
        error     : tuple

    where:
         retval contains dataset 'this' in data['this']
         and other information (e.g. locations etc) in
         retval.data and retval.names

    '''
    #self.store_header = goodfile.open().readline().close()
    this = np.genfromtxt(goodfile, skip_header=1)
    l = len(np.atleast_1d(this).shape)
    if this.size > 0:
        retval = ParamStorage()
        retval.name = ParamStorage()
        retval.data = ParamStorage()
        retval.data[dataset] = this
        retval.name.filename = goodfile
        retval.name.fmt = 'np.genfromtxt'
        return retval, (False, "")
    error = True
    error_msg = "Failed numpy read of %s" % goodfile
    return 0, (error, error_msg)
Exemplo n.º 16
0
def demonstration(conf='default.conf'):      
    '''
    Need to develop a new demo
    '''
    from eoldas_Lib import eoldas_setup
    from os import getcwd
    from eoldas_ParamStorage import ParamStorage

    options = ParamStorage()
    options.here = getcwd()
    options.logdir = 'logs'
    options.logfile = "logfile.log"
    options.datadir = ['.','~/.eoldas']
    #self = eoldas_setup('default.conf',options)            
    #return self        
    return True
Exemplo n.º 17
0
def safesplit(text, character):
    '''
    A function to split a string, taking account of [] and ()
    and quotes
    '''
    lent = len(text)
    intoken = ParamStorage()
    qtoken = ParamStorage()
    for i in ["'", '"']:
        qtoken[i] = False
    for i in ["()", "[]", "{}"]:
        intoken[i] = 0

    start = 0
    lst = []
    i = 0
    while i < lent:
        # are any of intoken, qtoken open
        isopen = False
        for (j, k) in intoken.iteritems():
            if text[i] == j[0]:
                intoken[j] += 1
            elif text[i] == j[1]:
                intoken[j] -= 1
            isopen = isopen or (intoken[j] != 0)
        for (j, k) in qtoken.iteritems():
            if text[i] == j[0]:
                qtoken[j] = not qtoken[j]
            isopen = isopen or qtoken[j]
        if text[i] == character and not isopen:
            lst.append(text[start:i])
            start = i + 1
        elif text[i] == '\\':
            i += 2
            continue
        i += 1
    lst.append(text[start:])
    return lst
Exemplo n.º 18
0
class eoldas_setup(object):
    """
    This is redundant

    """
    def __init__(self,datafile,options):
      
        self.options = options 
        # sort logging
        self.options.general.logdir         \
            = self.__getopt(self.options.general,'logdir',"logs")
        self.options.general.logfile        \
            = self.__getopt(self.options.general,'logfile',"logfile.log") 
        # 1.- set up logging for this particular run
        self.logger = set_up_logfile(self.options.general.logfile,\
                        name="eoldas_setup",logdir=self.options.general.logdir)

        try:
            self.setup()            
        except:
            self.logger.error("Unable to access critical elements of the options. See help(eoldas_setup.setup) for details")
            sys.exit(-1)
        # read conf file(s)
        self.configfile = data_file
        #config = ConfFile(self.configfile,dirs=dirs,log_name=self.logger)
        if len(config.infos) == 0:
            self.fail = True
            return
        # not sure what to do if multiple config files???
        # just take the first one at the moment
        self.config  = config.infos[0]
        # update with cmd line options
        self.config.update(self.options,combine=True)

        self.logger.info("Model sd scaling by %f over that defined in the config file" % self.config.general.model_sd)
        self.ok = self.process_config_file()

    def setup(self):
        '''
        Access and set up critical elements of the problem.

        This includes:
        
        The existence of:
            options.parameter.names (a list)

        '''
        options = self.options
        options.parameter.n_params = len(options.parameter.names)
        

        self.options = options

    def __getopt(self,options,key,value):
        if not hasattr(options,key):
            options[key] = value
        return options[key]

    def __pcheck(self,thisdict,name):
        '''
        Check that name exists in thisdict
        '''
        try:
            if name in thisdict.dict():
                return True
            else:
                return False
        except:
            return False

    def __min(self,a,b):
        '''
        Min utility for 2 numbers, ignoring None
        '''
        if a == None:
            out = b
        elif b == None:
            out = a
        else:
            out = np.min([a,b])
        if out == None:
            return 0
        else:
            return out     

        # the next critical thing is some observations
        obs = load_brdf_file (brf,self.config,bandpass_names={})
        if obs == False:
            return False

        self.config.operator.obs.update(obs,combine=True)

        # sets up an initial version of x_init
        # which is in the observation 'space' (ie one per obs)
        for n_par in xrange ( self.config.params.n_params ):
            #self.default_vals[n_par] = prior_mean[n_par]
            if np.all( self.obs.x_init[ :, n_par] == 0 ): # No
                self.obs.x_init [ :, n_par ] = self.default_vals [ n_par ]
        # try brfinit_files
        # which can overwrite x_init
        try:
            if self.options.preload != []:
                brfinit_files = self.options.preload
                self.brfinit_files['override'] = brfinit_files
        except:
            if self.options.preload != []:
                self.brfinit_files = ParamStorage ()
                self.brfinit_files['override'] = self.options.preload
                # this is a hack to get the same structure
                self.brfinit_files = self.brfinit_files.dict()
        thisdoys = None
        if self.brfinit_files is not None:
            # this is not consistent with having multiple files
            # and is a bit of a mess 
            for key in self.brfinit_files.keys():
                if type(self.brfinit_files[key]) == type([]):
                    initfile = self.brfinit_files[key][0]
                else:
                    initfile = self.brfinit_files[key]
                #(acovar, abandwidth, abands, anpt, anbands_max, alocation, \
                #    awhichfile, anbands, adoys, aqa, atheta_v, atheta_i,aphi_v, \
                #    aphi_i, aisobs, aobs, aobscovar, aparams_x) = \
                #    load_brdf_file(initfile)
                (thisdoys,thisparams) = self.read_parameters(initfile,confdir=confdir)
                # if fail, thisdoys is None
                #self.obs.x_init[:,:] = aparams_x[:,:]

        if thisdoys == None:
            self.brfinit_files = None
        # For convenience, we can invert the observation covariance matrices
        self.obs.obsinvcovar = []
        self.obs.real_obsinvcovar = []

        for sample_no in xrange( self.obs.npt ):
            temp_mtx = np.matrix( self.obs.obscovar[ sample_no ] ).I
            if self.config.params.scale_cost:
                self.logger.info ("Scaling obs by %f" % \
                    float(self.obs.npt*self.obs.nbands[0] ) )
                self.obs.obsinvcovar.append ( \
                        temp_mtx/float((self.obs.npt*self.obs.nbands[sample_no] )))
            else:
                self.obs.obsinvcovar.append( temp_mtx )
            self.obs.real_obsinvcovar.append (temp_mtx)    

        # if there is anything non zero in x_init, set params_x to that
        if self.obs.x_init.sum() > 0:
            self.params_x = self.obs.x_init.copy()
        else:
            self.params_x = np.zeros ((self.obs.npt, \
                                        self.config.params.n_params))
        # determine which params to fix, based primarily on solve_for flags
        fix_params = define_fixparams(self.parameters, \
            solve_for=self.solve_for,prior_sd=self.prior_sd,model_unc_cfg=self.model_unc_cfg)

        self.config.params.n_model_params = np.sum(fix_params==3) + np.sum(fix_params==4)

        # set up the grid based on the span of unique doys
        self.unique_doys, self.quantised_doys, self.obs_shift = quantise_time ( self.obs.doys, \
                                                self.time_quant ,grid=grid)
        self.grid_n_obs = self.unique_doys.shape[0]

       

        self.fix_params = np.tile(fix_params, self.grid_n_obs).reshape((self.grid_n_obs,self.config.params.n_params))

        self.logger.info ("%d days, %d quantised days" % ( len(self.unique_doys), \
            len(self.quantised_doys) ) )
        self.grid_n_params = fix_params.shape[0]

        # set up a grid model representation from self.params_x
        # we will use then when loading
        # self.params_x is a full representation in obs space
        # so we expand it to the model grid space
        self.store_params = self.get_x(self.params_x,self.fix_params*0.)
 
        # but this may contain zeros if a parameter has not been defined so should be set to the default value
        # or maybe interpolations is better
        udoys = np.unique(self.obs.doys)
        try:
            where_udoys = np.in1d(self.unique_doys,udoys)
        except:
            where_udoys = np.zeros_like(self.unique_doys).astype(np.bool)
            for i in udoys:
                w = np.where(self.unique_doys == i)
                where_udoys[w] = True
        for i in xrange(self.grid_n_params):
            self.store_params[:,i] = np.interp(self.unique_doys,self.unique_doys[where_udoys],self.store_params[where_udoys,i])
        
        # override this with data from brfinit_files
        if self.brfinit_files is not None:
            # zeroth ...
            # pull out elements of thisdoys that appear in  self.unique_doys
            
            # first interpolate thisparams onto the grid
            store_params = self.store_params*0.
            new_thisdoys = np.zeros( self.store_params.shape[0]).astype(np.int)
            # loop over thisdoys and load where appropriate
            for (i,j) in enumerate(thisdoys):
                ww = np.where(j == self.unique_doys)
                store_params[ww,:] = thisparams[i,:] 
                new_thisdoys[ww] = j
            thisdoys = new_thisdoys
            udoys = np.unique(thisdoys)
            try:
                where_udoys = np.in1d(thisdoys,udoys)
            except:
                where_udoys = np.zeros_like(thisdoys).astype(np.bool)
                for i in udoys:
                    w = np.where(where_udoys  == i)
                    where_udoys[w] = True
            for i in xrange(self.grid_n_params):
                self.store_params[:,i] = np.interp(self.unique_doys,self.unique_doys[where_udoys],store_params[where_udoys,i])

        # deal with model uncert
        self.model_unc = np.ones((self.fix_params.shape[1])) 
        for ( i, k ) in enumerate ( self.parameters ):
            if self.model_unc_cfg [ k ] > 0:
                self.model_unc[i] = self.model_unc[i] * self.model_unc_cfg [ k ]
        self.prior_m = np.array([self.prior_mean[k] for k in self.parameters ])
        self.prior_std = np.array([self.prior_sd[k] for k in self.parameters ])

        return #( prior_mean, prior_sd, model_unc, abs_tol, scale_cost)

    def get_x( self, x_obs,  x_model, summer=False):
        """
        return an x_model representation which has parameter values for the 
        complete model grid. The array x_obs has a representation of the 
        parameter values only for observation points, whereas x_model is 
        typically defined over the whole assimilation period/region.

        When loading parameters in this way (from observation space
        to model space, only the parameter associated with the first 
        observation at a particular point is taken (summer=False)

        When loading derivatives (e.g. when using the adjoint) we need
        to sum over all observation grid points (summer=True)
        
        Parameters
        -----------
        x_obs : array-like
            The state vector representation that corresponds to the observations
        x_model : array-like
            The state vector representation that corresponds to the assimilation
            interval.
        """
        if summer == False:
            for i in np.unique(self.obs_shift).astype(np.int):
                w = np.where(self.obs_shift == i)[0][0]
                x_model[i,:] = x_obs[w,:]
        else:
            x_model[:,:] = 0.
            for i in np.unique(self.obs_shift).astype(np.int):
                w = np.where(self.obs_shift == i)[0]
                for j in w:
                    x_model[i,:] = x_model[i,:] + x_obs[j,:]
        return x_model

    def write_parameters(self,filename,params,ofmt='ASCII'):
        """
        Write the parameters out to filename
        """
        if ofmt == 'ASCII':
            self.logger.info ( "Saving parameters to %s" % filename)
            fp = open(filename,'w')
            fp.write("# PARAMETERS %s\n" % "".join ( [ "%s " % i  for i in self.parameters])) 
            for i in xrange(self.grid_n_obs):
                 fp.write("%f %s\n" % (self.unique_doys[i],"".join ( [ "%s " % j  for j in params[i,:]])))
            fp.close()
Exemplo n.º 19
0
    def read_single_conf_file(self, conf_files, options=None):
        """
    
        Purpose:
        parse the information from conf_files into a 
        ConfigParser class instance and return this.
    
        Parameters:
            conf_files : list of one or more config files
    
        Options:
            options=None    : pass an options structure through
        
        Uses:
            self.datadir=['.',,'~/.eoldas'] 
                                 :  list of directories to look 
                                    for config files
            self.env=None        :  name of an environment variable 
                                    where config files
                                    can be searched for if not found 
                                    in datadir (or absolute 
                                    path name not given)
            self.fatal=False     :  flag to state whether the 
                                    call should fail if 
                                    a requested config file is not found.
    
    
        Returns:
            tuple           : (config, config_error)
        where:
            config          : ConfigParser class instance
                              or False if an error occurs
            config_error    : string giving information on error
        """
        import ConfigParser
        from eoldas_Lib import get_filename
        # Instantiate a parser
        config = ConfigParser.ConfigParser()
        # Read the config files. If it doesn't exist, raise exception.
        #
        if type(conf_files) == str:
            conf_files = [conf_files]
        all_conf_files = []

        for fname in conf_files:
            fname,fname_err =  get_filename(fname,datadir=self.datadir,\
                                            env=self.env)
            if fname_err[0] != 0:
                if self.fatal:
                    return False,False,\
                        "Cannot find configuration file %s\n%s" \
                        % (fname,fname_err[1])
            else:
                all_conf_files.append(fname)
                thisdir = os.path.dirname(fname)
                if not thisdir in self.datadir:
                    self.datadir.append(thisdir)
        if len(all_conf_files) == 0:
            return False,False,\
                "%s: No valid conf files found in list %s in dirs %s" \
                % (os.getcwd(),conf_files,self.datadir)
        config.config_files = config.read(all_conf_files)
        if len(config.config_files) == 0:
            return False,False,\
                "%s: No valid conf files found in list %s in dirs %s" \
                % (os.getcwd(),conf_files,self.datadir)

        # from here on, we attempt to pull specific information from
        # the conf files
        info = ParamStorage(name='info',doc=\
                    'Configuration information for %s' % \
                    str(config.config_files))

        # scan everything into config.info
        # but it helps to sort it to get the info in the right order
        sections = config.sections()
        #sections.sort()
        firstsections = []
        secondsections = []
        for this in sections:
            if this[:7] == 'general' or this[:9] == 'parameter':
                firstsections.append(this)
            else:
                secondsections.append(this)
        firstsections.sort()
        sections = firstsections
        [sections.append(i) for i in secondsections]
        for this in sections:
            self.logger.debug('...Section %s' % this)
            self.scan_info(config, this, info, this, info)
        self.rescan_info(config, this, info, this, info, 0)

        self.config = config
        self.info = info
        if options != None and type(options) == ParamStorage:
            self.info.update(options, combine=True)

        # sort any helper text looping over self.info
        # into self.loaders
        self.__sort_help(self.info, "")
        try:
            self.logger.info("Config: %s read correctly" \
                       % str(all_conf_files))
        except:
            pass
        return self.config,self.info,"Config: %s read correctly" \
                % str(all_conf_files)
Exemplo n.º 20
0
    def parseLoader(self,loaders):
        """
        Utility to load a set of terms from the list loaders 
        into the ParamStorage general

        If there are 3 terms in each loaders element, they refer to:
        1. name
        2. default value
        3. helper text
        If there is a fourth, it is associated with extras 
        (short parser option)
        """
        general = ParamStorage ()
        general.__default__ = ParamStorage ()
        general.__extras__ = ParamStorage ()
        general.__helper__ = ParamStorage ()

        for this in loaders:
            if len(this) > 1:
                general[this[0]] = this[1]
                general.__default__[this[0]] = this[1]
            else:
                general[this[0]] = None
                general.__default__[this[0]] = None
            if len(this) > 2:
                general.__helper__[this[0]] = this[2]
            else:
                general.__helper__[this[0]] = optparse.SUPPRESS_HELP
            if len(this) > 3:
                general.__extras__[this[0]] = "%s" % this[3]
            else:
                general.__extras__[this[0]] = None
            # make sure arrays arent numpy.ndarray
            if type(general.__default__[this[0]]) == np.ndarray:
                general.__default__[this[0]] = \
                        list(general.__default__[this[0]])

        self.top.update(self.__unload(general),combine=True)
Exemplo n.º 21
0
    def __init__(self,
                 args,
                 name=None,
                 general=None,
                 log=False,
                 logger=None,
                 outdir=".",
                 getopdir=False,
                 parse=True):
        """
        Initialise parser class.

        This sets up the class defaults.
        
        Options:
            
        general=general: this over-rides and defaults with values 
                        set in parser general can be of the form:
                         1. class ParamStorage (i.e. the same form 
                            as self.general)
                         2. a command line list (where the first 
                            item in the list is ignored
                         3. a string containing a set of command line general
            
                         See self.parse() for more details on general 
                            2 and 3 as these simply make a call to tha
                            method.
            
        log=True         If log is set to True, then logging starts 
                            when this class is instanced.
                            Note that the logfile and logdir might 
                            change if subsequent calls to Parser.parse()
                            are made
        """
        if type(args) == str:
            args = args.split()
        self.dolog = log
        self.log = log
        self.name = args[0]
        self.args = args[1:]
        self.fullargs = args
        self.store_fullargs = args
        if name == None:
            import time
            thistime = str(time.time())
            name = type(self).__name__
            name = "%s.%s" % (name, thistime)
        self.thisname = name
        # find the following flags:
        # --conf | -c : conf
        # --datadir   : datadir
        datadir = [".","~/.eoldas",sys.path[0]+'/../bin',sys.path[0]+'/../confs',\
  sys.path[0]+'/../system_confs',sys.path[0]+'/../eoldaslib']
        conf = "default.conf"
        logfile = None
        logdir = "."
        self.top = ParamStorage()
        self.top.general = ParamStorage()
        self.top.general.__helper__ = ParamStorage()
        self.top.general.__default__ = ParamStorage()
        self.top.general.__extras__ = ParamStorage()
        self.top.general.conf = []
        for i in xrange(len(self.args)):
            theseargs = self.args[i].split('=')
            if theseargs[0] == "--conf":
                conf = theseargs[1]
                self.top.general.conf.append(conf)
            elif theseargs[0][0:2] == "-c":
                if len(theseargs) > 2:
                    conf = theseargs[0][2:]
                else:
                    conf = self.args[i + 1]
                self.top.general.conf.append(conf)
            elif theseargs[0] == "--datadir":
                datadir1 = theseargs[1].replace('[','').\
                                            replace(']','').split()
                [datadir1.append(datadir[i]) for i in \
                                            xrange(len(datadir))]
                datadir = datadir1
            elif theseargs[0] == "--logfile":
                logfile = theseargs[1]
            elif theseargs[0] == "--logdir":
                logdir = theseargs[1]
            elif theseargs[0] == "--outdir":
                outdir = theseargs[1]
        if self.top.general.conf == []:
            self.top.general.conf = conf
        if logfile == None:
            logfile = conf.replace('conf', 'log')
        self.top.general.here = os.getcwd()
        self.top.general.datadir = datadir
        self.top.general.logfile = logfile
        self.top.general.logdir = logdir
        self.top.general.outdir = outdir
        # add here to datadir
        # in addition to '.' to take account of the change of directory
        self.top.general.datadir = self.__add_here_to_datadir(\
                    self.top.general.here,self.top.general.datadir)
        self.top.general.datadir = self.__add_here_to_datadir(\
                    self.top.general.outdir,self.top.general.datadir)
        # cd to where the output is to be
        self.__cd(self.top.general.outdir)
        # set up the default command line options
        self.default_loader()
        # update with anything passed here
        if general and type(general) == ParamStorage:            self.top.update(\
self.__unload(general),combine=True)
        # read the conf files to get any cmd line options
        self.logger = sortlog(self,self.top.general.logfile,logger,name=self.thisname,\
                    logdir=self.top.general.logdir)
        self.config = ConfFile(self.top.general.conf,name=self.thisname+'.config',\
      loaders=self.loaders,datadir=self.top.\
                    general.datadir,logger=self.logger,logdir=self.top.general.logdir,\
                    logfile=self.top.general.logfile)
        if len(self.config.configs) == 0:
            this = "Warning: Nothing doing ... you haven't set any configuration",\
      self.config.storelog
            try:
                self.logger(this)
            except:
                print "Called with args:"
                print "eoldas", self.args
                pass
            raise Exception(this)

        # now loaders contains all of the defaults set here
        # plus those from the config (config opver-rides defaults here)
        self.loaders = self.config.loaders
        # now convert loaders into parser information
        self.parseLoader(self.loaders)
        self.parse(self.fullargs)
        if general and type(general) == ParamStorage:
            self.top.update(self.__unload(general), combine=True)
        if general and type(general) == str:
            self.parse(general.split())
        if general and type(general) == list:
            self.parse(general)
        # now update the info in self.config
        for i in self.config.infos:
            i.update(self.top, combine=True)
            # so now all terms in self.config.infos
            # contain information from the config file, updated by
            # the cmd line
            i.logger = self.logger
            i.log()
        # move the information up a level
        self.infos = self.config.infos
        self.configs = self.config.configs
        self.config_log = self.config.storelog

        #if getopdir:
        #    self.sortnames()
        self.config.loglist(self.top)

        #del(self.config.infos)
        #del(self.config.configs)
        #del(self.config)
        self.__cd(self.top.general.here)
Exemplo n.º 22
0
 def __setattr__(self,this,value):
     '''
     Variable setting method for style self.this
         
     Varies what it does depending on the type of value.
         
     The method interprets and sets the SpecialVariable value:
         
         1.  ParamStorage or SpecialVariable. The data are directly loaded.
             This is one of the most flexible formats for input. It expects
             fields 'data' and/or 'name', which are loaded into self.
             There will normally be a field data.this, where this is 
             the variable name passed here.
         2.  A dictionary, same format as the ParamStorage.
         3.  A tuple, interpreted as (data,name) and loaded accordingly.
         4.  *string* as filename (various formats). An attempt to read the
             string as a file (of a set of formats) is made. If none pass 
             then it it maintained as a string.
         5.  A numpy array (np.array) that is loaded into self.data.this.
         6.  Anything else. Loaded into self.data.this as a numpy array.
         
     '''
     if self.simple:
         self.set(this,value)    
         return
     t = type(value)
     try:
         if t == ParamStorage or t == SpecialVariable:
             # update the whole structure
             #self.__set_if_unset('data',ParamStorage())
             #self.__set_if_unset('name',ParamStorage()) 
             self.data.update(value.data,combine=True)
             self.name.update(value.name,combine=True)
         elif t == dict:
             n_value = ParamStorage().from_dict(value)
             self.__setattr__(this,n_value)
         elif t == tuple or t == list:
             # assumed to be (data,name) or [data,name]
             #self.__set_if_unset('data',ParamStorage())
             #self.__set_if_unset('name',ParamStorage())
             #ParamStorage.__setattr__(self['data'],this,value[0]) 
             #ParamStorage.__setattr__(self['name'],this,value[1])
             ParamStorage.__setattr__(self['data'],this,np.array(value))
         elif t == str:
             # set the term
             #self.__set_if_unset('data',ParamStorage())
             #self.__set_if_unset('name',ParamStorage())
             ParamStorage.__setattr__(self['data'],this,value)
             # interpret as a file read if possible
             self.process_data_string(this,info=self.info)
         elif t == np.ndarray:
             #self.__set_if_unset('data',ParamStorage())
             #self.__set_if_unset('name',ParamStorage())
             ParamStorage.__setattr__(self['data'],this,value)
         else:
             ParamStorage.__setattr__(self['data'],this,\
                                      np.array(value))
     except:
         if self.logger:
             self.logger.info("Failed to set SpecialVariable %s from %s %s"\
                          %(this,t.__name__,value))
         return
     if self.logger:
         self.logger.info("Set variable %s from type %s"%(this,t.__name__))
Exemplo n.º 23
0
    def prep(self, thisconf):
        '''
        A method to prepare the solver
            
        '''
        self.root = self.confs.root[thisconf]
        root = self.confs.root[thisconf]

        self.sortMask()

        self.op = sortopt(root.general, 'optimisation', ParamStorage())
        self.op.plot = sortopt(self.op, 'plot', 0)
        self.op.name = sortopt(self.op, 'name', 'solver')
        self.op.maxfunevals = sortopt(self.op, 'maxfunevals', 2e4)
        self.op.maxiter = sortopt(self.op, 'maxiter', 1e4)
        self.op.gtol = sortopt(self.op, 'gtol', 1e-3)
        self.op.iprint = sortopt(self.op, 'iprint', 1)
        self.op.solverfn = sortopt(self.op, 'solverfn', 'scipy_lbfgsb')
        self.op.randomise = sortopt(self.op, 'randomise', False)
        self.op.no_df = sortopt(self.op, 'no_df', False)

        self.result = sortopt(root.options, 'result', ParamStorage())
        self.result.filename = sortopt(root.options.result,'filename',\
                                       'results.pkl')
        self.result.fmt = sortopt(root.options.result, 'format', 'pickle')
        try:
            self.transform = self.root.options.x.transform
            self.invtransform = self.root.options.x.transform
        except:
            self.transform = None
            self.invtransform = None

        # descend into the operators and identify any observation ops
        # we then want to be able to write out files (or plot data)
        # that are H(x).
        # We only do this for Operators that have both 'x' and 'y'
        # terms as we store the filename under 'y.result'
        self.Hx_ops = []
        for i, op in enumerate(root.operators):
            op.loader(root)
            if 'y_meta' in op.dict() and op.options.y.datatype == 'y':
                # There is a potential observation
                op.y_state.options.y.result = \
                    sortopt(op.y_state.options.y,'result',ParamStorage())
                op.y_state.options.y.result.filename = \
                    sortopt(op.y_state.options.y.result,\
                        'filename',op.y_state.thisname)
                op.y_state.options.y.result.format = \
                    sortopt(op.y_state.options.y.result,\
                        'format','PARAMETERS')
                state = op.y_state._state
                this = { \
                    'filename':op.y_state.options.y.result.filename,\
                    'format':op.y_state.options.y.result.format,\
                    'state':state,\
      'y':op.y_state,\
                    'op':op,\
      'transform':self.transform,\
      'invtransform':self.invtransform,\
                    'Hx':op.linear.H}
                op.Hx_info = this
                self.Hx_ops.append(this)
            else:
                this = { \
                    'transform':self.transform,\
                    'invtransform':self.invtransform,\
                }
                op.Hx_info = this
Exemplo n.º 24
0
def read_numpy(self, filename, name, info=[]):
    '''
        Try to read the file as as a NpzFile file
        '''
    from eoldas_Lib import set_default_limits,check_limits_valid,\
        quantize_location,dequantize_location

    # none of these ciritical to functioning
    try:
        info = self._state.info
    except:
        info = []
    try:
        names = self.name.state
    except:
        try:
            names = self.Name.state
        except:
            names = None
    try:
        control = self.Name.control
    except:
        try:
            control = self.name.control
        except:
            control = None
    try:
        location = self.name.location
    except:
        try:
            location = self.Name.location
        except:
            location = ['time', 'row', 'col']
    try:
        limits = self.name.qlocation
    except:
        try:
            limits = self.Name.qlocation
        except:
            limits = set_default_limits(location)
    # refl_check=False,names=None,\
    # control=['mask','vza','vaa','sza','saa'],\
    # location=['time','row','col'],limits=None

    # location specifies the dimesions and names of the
    # problem, e.g., & typically [time,row,col]
    limits = np.array(check_limits_valid(limits))

    try:
        f = np.load(filename)
        if not type(f).__name__ == 'NpzFile':
            f.close()
            self.error_msg = "%s is not a NpzFile" % filename
            self.error = True
            if 'logger' in self or 'logger' in self.dict():
                self.logger.info(self.error_msg)
            return 0, (self.error, self.error_msg)
    except:
        self.error_msg = "a problem opening %s as a NpzFile" % filename
        self.error = True
        if 'logger' in self or 'logger' in self.dict():
            self.logger.info(self.error_msg)
        return 0, (self.error, self.error_msg)
    # ok so far then
    # lets have a look inside

    ncontents = np.array(f.files)
    contents = np.array(f.files)
    # translation table for default names
    def_names = 'b1 b2 b3 b4 b5 b6 b7'.split()
    if names == None:
        # assume MODIS
        names = def_names
    def_alt_names = \
        '645.5 856.5 465.6 553.6 1241.6 1629.1 2114.1'.split()
    # look for any of names in contents
    datasets = []
    alt_datasets = []
    alt_names = names
    for i in xrange(len(np.atleast_1d(contents))):
        if contents[i] in names:
            datasets.append(i)

    if not len(np.atleast_1d(datasets)):
        if 'logger' in self or 'logger' in self.dict():
            self.logger.error(\
                          "None of requested datasets %s found in %s ..." \
                          %(str(names),filename) + \
                          " trying default MODIS names: only %s"\
                          %(str(contents)))
        names = def_names
        alt_names = def_alt_names
        for i in xrange(len(np.atleast_1d(contents))):
            if contents[i] in names:
                datasets.append(i)
        if not len(np.atleast_1d(datasets)):
            self.error_msg = "None of requested datasets %s found in %s"\
                %(str(names),filename) + ' ' + \
                "... trying default MODIS names: only %s"\
                %(str(contents))
            self.error = True
            if 'logger' in self or 'logger' in self.dict():
                self.logger.error(self.error_msg)
            return 0, (self.error, self.error_msg)
    trans_names = {}
    for (i, j) in enumerate(alt_names):
        trans_names[names[i]] = j


#trans_names = {names[i]:j for (i,j) in enumerate(alt_names)}
    alt_name = []
    this_name = []
    for i in datasets:
        this_name.append(contents[i])
        alt_name.append(trans_names[contents[i]])

    # Translate  some old stylies...
    trans = {'raa': 'vaa', 'doys': 'time'}
    for i in trans:
        if i in contents:
            ncontents[np.where(contents == i)[0]] = trans[i]
    # as a minimum, there needs to be some definition of one of
    # the terms in location

    # check how many dimensions this has
    # now find a dataset
    try:
        # This could be more general, but this will do for now as its useful
        # for spatial datasets
        QA_OK = np.array(\
                         [8, 72, 136, 200, 1032, 1288, 2056,2120, 2184, 2248])
        doy = f['doys'] - 2004000
        qa = f['qa']
        vza = f['vza']
        sza = f['sza']
        raa = f['raa']
        y = []
        for i in this_name:
            y.append(f[i])
        #mask = np.logical_or.reduce([qa==x for x in QA_OK ])
        if 'logger' in self or 'logger' in self.dict():
            self.logger.info(\
                         "sucessfully interpreted NpzFile dataset from %s"\
                             %filename)
            self.logger.info("sub-setting ...")
        controls = []
        locations = []
        grid = []
        qlocations = []
        thisshape = vza.shape
        starter = {'time': np.min(doy), 'row': 0, 'col': 0}
        delta = {'time': 1, 'row': 1, 'col': 1}
        if len(np.atleast_1d(limits)) < 3:
            from eoldas_Lib import set_default_limits
            old_loc = location
            location = np.array(['time', 'row', 'col'])
            lim2 = set_default_limits(location)
            for i in xrange(len(np.atleast_1d(limits))):
                ww = np.where(old_loc[i] == location)[0]
                lim2[ww] = list(limits[i])
            limits = lim2
        for i in xrange(len(np.atleast_1d(limits))):
            if limits[i][0] == None:
                limits[i][0] = starter[location[i]]
            if limits[i][1] == None:
                limits[i][1] = (thisshape[i] - 1) + starter[location[i]]
            if limits[i][2] == None:
                limits[i][2] = delta[location[i]]
        limits = np.array(limits)
        start_doy = limits[0][0]
        end_doy = limits[0][1]
        step_doy = limits[0][2]
        start_row = limits[1][0]
        end_row = limits[1][1]
        step_row = limits[1][2]
        start_col = limits[2][0]
        end_col = limits[2][1]
        step_col = limits[2][2]
        gooddays = np.logical_and.reduce(np.concatenate(\
                            ([doy >= start_doy],[doy  <=end_doy])))
        qa = qa[gooddays, start_row:end_row + 1, start_col:end_col + 1]
        vza = vza[gooddays, start_row:end_row + 1,
                  start_col:end_col + 1] * 0.01
        sza = sza[gooddays, start_row:end_row + 1,
                  start_col:end_col + 1] * 0.01
        raa = raa[gooddays, start_row:end_row + 1,
                  start_col:end_col + 1] * 0.01
        yy = []
        for i in xrange(len(np.atleast_1d(this_name))):
            this = y[i]
            yy.append(this[gooddays,start_row:end_row+1,\
                           start_col:end_col+1]*0.0001)
        doy = doy[gooddays]
        # now do QA
        mask = np.zeros_like(qa).astype(bool)
        # loop over qa
        for j in xrange(len(np.atleast_1d(QA_OK))):
            ww = np.where(qa == QA_OK[j])
            mask[ww] = True
        # better look over data to check valid
        for j in xrange(len(np.atleast_1d(yy))):
            ww = np.where(yy[j] < 0)
            mask[ww] = False
        ww = np.where(mask)
        if 'logger' in self or 'logger' in self.dict():
            self.logger.debug('parsing dataset: %d samples look ok'\
                          %np.array(ww).shape[1])
        vza = vza[ww]
        sza = sza[ww]
        raa = raa[ww]
        doy = doy[ww[0]]
        row = ww[1] + start_row
        col = ww[2] + start_col
        locations = np.array([doy, row, col])
        nnn = len(np.atleast_1d(locations[0]))
        orig = np.repeat(np.array([start_doy, start_row, start_col]),
                         locations.shape[1]).reshape(locations.shape).T
        div = np.repeat(np.array([step_doy, step_row, step_col]),
                        locations.shape[1]).reshape(locations.shape).T
        qlocations = ((locations.T - orig) / div.astype(float)).astype(int).T
        controls = np.array([np.ones_like(doy).astype(bool),\
                             vza,raa,sza,0*doy])
        y = []
        for i in xrange(len(np.atleast_1d(this_name))):
            this = yy[i]
            y.append(this[ww])
        grid = np.array(y)
        fmt = 'BRDF-UCL'
        control = ['mask', 'vza', 'vaa', 'sza', 'saa']
        bands = alt_name
        if not np.array(grid).size:
            if 'logger' in self or 'logger' in self.dict():
                self.logger.error(\
                              "Warning: returning a zero-sized dataset ... "+\
                              " I wouldn;t try to do anything with it")
        # in case we dont have data for all bands
        mask =  np.logical_or.reduce([[this_name[i]==x for x in names] \
                                      for i in xrange(len(np.atleast_1d(this_name)))])
        sd = np.array('0.004 0.015 0.003 0.004 0.013 0.01 0.006'\
                      .split())[mask]
        sd = np.array([float(i) for i in sd.flatten()])\
            .reshape(sd.shape)
        nsamps = grid.shape[1]
        sd = sd.repeat(nsamps).reshape(grid.shape).T
        datasets = ParamStorage()
        datasets.data = ParamStorage()
        datasets.name = ParamStorage()
        datasets.name.fmt = fmt
        grid = grid.T
        datasets.data[name] = np.zeros([grid.shape[0],len(np.atleast_1d(names))])\
                                                        .astype(object)
        datasets.data[name][:, :] = None
        for i in xrange(len(np.atleast_1d(this_name))):
            ww = np.where(names == this_name[i])[0][0]
            datasets.data[name][:, ww] = grid[:, i]
        datasets.data.location = np.array(locations).T
        datasets.data.control = np.array(controls).T
        datasets.data.qlocation = np.array(qlocations).T
        datasets.name[name] = np.array(names)
        datasets.name.location = np.array(['time', 'row', 'col'])
        datasets.name.control = np.array(control)
        datasets.name.qlocation = limits
        datasets.name.bands = np.array(bands)
        datasets.data.sd = np.zeros([grid.shape[0],len(np.atleast_1d(names))])\
                                                        .astype(object)
        # for i in xrange(grid.shape[0]):
        # datasets.data.sd[i,:] = self.options.sd
        datasets.data.sd[:, :] = None
        for i in xrange(len(np.atleast_1d(this_name))):
            ww = np.where(names == this_name[i])[0][0]
            datasets.data.sd[:, ww] = sd[:, i]
        datasets.name.sd = np.array(names)
        if 'logger' in self or 'logger' in self.dict():
            self.logger.debug('finished parsing dataset')
    except:
        self.error_msg=\
            "a problem processing information from  %s as a NpzFile"\
                %filename
        self.error = True
        if 'logger' in self or 'logger' in self.dict():
            self.logger.info(self.error_msg)
        return 0, (self.error, self.error_msg)
    f.close()
    if 'logger' in self or 'logger' in self.dict():
        self.logger.info('... done')
    self.error = False
    self.error_msg = ""
    return datasets, (self.error, self.error_msg)
Exemplo n.º 25
0
def demonstration():
    from eoldas_State import State
    from eoldas_ParamStorage import ParamStorage
    import numpy as np
    
    # a basic set up for State, setting names & bounds etc
    options = ParamStorage()
    options.logfile = 'test/data_type/logs/log.dat'
    options.names = \
       'gamma xlai xhc rpl xkab scen xkw xkm xleafn xs1 xs2 xs3 xs4 lad'.split()
    options.bounds = [[0.01,None],\
                      [0.01,0.99],\
                      [0.01,10.0],\
                      [0.001,0.10],\
                      [0.1,0.99],\
                      [0.0,1.0],\
                      [0.01,0.99],\
                      [0.3,0.9],\
                      [0.9,2.5],\
                      [0.0, 4.],\
                      [0.0, 5.],\
                      [None, None],\
                      [None, None],\
                      [None, None]]
    options.default = -1.0*np.ones(len(options.names))
    options.location = 'time'.split()
    options.control = 'mask vza vaa sza saa'.split()
    options.datadir = ['.','test/data_type']

    name = "eoldas_data_type test 0"
    options.limits = [[170,365,1]]

    self = State(options,datatype='y',name=name,datadir=\
                             options.datadir,env=None,logfile=options.logfile)
    self.tester()
            
    # Now we set some state data        
    this = ParamStorage()
    # how many state vector elements should there be?
    n_states = len(self.Name.state)       
    self.state  = np.ones([100,n_states])
    self.Data.sd = np.ones([1,n_states])
    self.Name.sd = self.Name.state
  
    print '******************'
    print (self.Data.sd,self.Name.sd)

    
    this.data = ParamStorage() 
    this.name = ParamStorage() 
    this.data.state = self.state *2
    controls = self.Name.control
    n_controls = len(controls)
    this.data.control = np.ones([100,n_controls])
    this.data.location = np.ones([100,n_controls])

    # we can load x_state from a ParamStorage
    self.state = this
    # now we should see the control data etc.
    self.tester()

    # change a dataset name to see if that works:
    # should load everything as numpy arrays
           
    self.Name.control = np.array(['vza','sza'])

    # change a dataset to see if that works .. deliberately load a bad one
    self.Data.control = 0

    # now try a direct state data load
    self.state = np.zeros([100,100])+5.
    # which will load into self.Data.state

    self['state'] = np.zeros([100,100])+6.
    #now try accessing it:
    print self.state

    # change the control info
    self.Name.control = np.array(['vza'])
    print self.Name.control

    # reset it 
    self.state = self.state * 2
    print self.state
    # now try reading a file into state
    self.state = 'test/data_type/input/test.brf'
    print '========'
    print 'data from a BRDF file'
    self.tester()
    print '========'
    # written as a pickle
    self.write('test/data_type/output/test.pickle',None,fmt='pickle')
    self.logger.info("...DONE...") 
    name = "eoldas_data_type test 1"
    del self
    self1 = State(options,datatype='y',name=name,datadir=\
             options.datadir,env=None,logfile=options.logfile,grid=True)
    # read from pickle
    self1.state = 'test/data_type/output/test.pickle'
    print '========'
    print 'data from a pickle file'
    self1.tester()
    print '========'
    # try to load an npx file
    del self1
    options.location = 'time row col'.split()
    options.limits = [[170,365,1],[0,500,1],[200,200,1]]

    self2 = State(options,datatype='y',name=name,datadir=\
                  options.datadir,env=None,logfile=options.logfile)
    self2.state = 'test/data_type/input/interpolated_data.npz'
    print '========'
    print 'data from a npz file'
    self2.tester()
    print '========'
    # write as BRDF-UCL
    self2.write('test/data_type/output/test.brf',None,fmt='BRDF-UCL')
    del self2
    self3 = State(options,datatype='y',name=name,datadir=\
             options.datadir,env=None,logfile=options.logfile)
    # then test the reader
    self3.state = 0.
    self3.state = 'test/data_type/output/test.brf'
    print '========'
    print 'data from a BRDF-UCL file'
    print '========'
    self3.tester()
    print '========'
    # then write as a PARAMETERS file
    self3.write('test/data_type/output/test.param',None,fmt='PARAMETERS')
    del self3
    options.location = 'time row col'.split()
    options.limits = [[170,365,1],[0,500,1],[200,200,1]]
    options.control = np.array(['mask','vza','vaa','sza','saa']) 
    self4 = State(options,datatype='y',name=name,datadir=\
             options.datadir,env=None,logfile=options.logfile)
    # then test the reader
    self4.state = 0.
    self4.state = 'test/data_type/output/test.param'
    print '========'
    print 'data from a PARAMETERS file'
    print '========'
    self4.tester()
    print '========'
Exemplo n.º 26
0
    def _load_rt_library(self, rt_library):
        """
        A method that loads up the compiled RT library code and sets some 
        configuration options. This method tries to import all the methods and
        make them available through `self.rt_library.<method_name>`. It is also
        a safe importer: if some functions are not available in the library,
        it will provide safe methods from them. Additionally, while importing,
        a number of configuration options in the class are also updated or set 
        to default values.
        
        Parameters
        -----------
        rt_library : string
        This is the name of the library object (.so file) that will be 
        loaded
            
            """
        from eoldas_Lib import sortopt
        import_string = "from %s import " % (rt_library)
        self.logger.debug("Using %s..." % rt_library)

        self.rt_library = sortopt(self, 'rt_library', ParamStorage())
        # 1. Import main functionality
        try:
            self.logger.debug("Loading rt_model")
            exec(import_string + "rt_model")
            self.rt_library.rt_model = rt_model
        except ImportError:
            self.logger.info(\
                "Could not import basic RT functionality: rt_model")
            self.logger.info(\
                "Check library paths, and whether %s.so is available" % \
                rt_library)
            raise Exception('error importing library %s' % rt_library)

        # 1a. Import conditioning methods that can be ignored
        try:
            exec(import_string + 'rt_modelpre')
            self.rt_library.rt_modelpre = rt_modelpre
        except:
            self.rt_library.rt_modelpre = self._nowt
        try:
            exec(import_string + 'rt_modelpre')
            self.rt_library.rt_modelpost = rt_modelpre
        except:
            self.rt_library.rt_modelpost = self._nowt

        # 2. Try to import derivative
        self.rt_model.ignore_derivative = sortopt(self.rt_model,\
                                'ignore_derivative',False)

        if self.rt_model.ignore_derivative == False:
            try:
                exec ( import_string + \
                    "rt_modeld, rt_modeldpre, rt_modeldpost" + \
                    ", rt_modelpred" )
                self.rt_model.have_adjoint = True
                self.J_prime = self.J_prime_full
                self.rt_library.rt_modeld = rt_modeld
                self.rt_library.rt_modeldpre = rt_modeldpre
                self.rt_library.rt_modeldpost = rt_modeldpost
                self.rt_library.rt_modelpred = rt_modelpred
            except ImportError:
                self.logger.info(
                    "No adjoint. Using finite differences approximation.")
                self.rt_model.have_adjoint = False
        else:
            self.logger.info(
                "ignoring adjoint. Using finite differences approximation.")
            self.rt_model.have_adjoint = False

        self._configure_adjoint()

        try:
            exec(import_string + "rt_model_deriv")
            self.rt_library.rt_model_deriv = rt_model_deriv
        except ImportError:
            self.rt_library.rt_model_deriv = None
Exemplo n.º 27
0
def read_numpy(self,filename,name,info=[]):
    '''
        Try to read the file as as a NpzFile file
        '''
    from eoldas_Lib import set_default_limits,check_limits_valid,\
        quantize_location,dequantize_location
    
    # none of these ciritical to functioning
    try:
        info = self._state.info
    except:
        info = []
    try:
        names = self.name.state
    except:
        try:
            names = self.Name.state
        except:
            names = None
    try:
        control = self.Name.control
    except:
        try:
            control = self.name.control
        except:
            control = None
    try:
        location = self.name.location
    except:
        try:
            location = self.Name.location
        except:
            location = ['time','row','col']
    try:
        limits = self.name.qlocation
    except:
        try:
            limits = self.Name.qlocation
        except:
            limits = set_default_limits(location)
    # refl_check=False,names=None,\
    # control=['mask','vza','vaa','sza','saa'],\
    # location=['time','row','col'],limits=None
    
    # location specifies the dimesions and names of the
    # problem, e.g., & typically [time,row,col]
    limits = np.array(check_limits_valid(limits))
    
    try:
        f = np.load(filename)
        if not type(f).__name__ == 'NpzFile':
            f.close()
            self.error_msg="%s is not a NpzFile"%filename
            self.error=True
            if 'logger' in self or 'logger' in self.dict():
                self.logger.info(self.error_msg)
            return 0,(self.error,self.error_msg)
    except:
        self.error_msg="a problem opening %s as a NpzFile"%filename
        self.error=True
        if 'logger' in self or 'logger' in self.dict():
            self.logger.info(self.error_msg)
        return 0,(self.error,self.error_msg)
    # ok so far then
    # lets have a look inside
    
    ncontents = np.array(f.files)  
    contents = np.array(f.files)
    # translation table for default names
    def_names = 'b1 b2 b3 b4 b5 b6 b7'.split()
    if names == None:
        # assume MODIS
        names = def_names
    def_alt_names = \
        '645.5 856.5 465.6 553.6 1241.6 1629.1 2114.1'.split()
    # look for any of names in contents
    datasets = []
    alt_datasets = []
    alt_names = names
    for i in xrange(len(np.atleast_1d(contents))):
        if contents[i] in names:
            datasets.append(i)
    
    if not len(np.atleast_1d(datasets)):
        if 'logger' in self or 'logger' in self.dict():
            self.logger.error(\
                          "None of requested datasets %s found in %s ..." \
                          %(str(names),filename) + \
                          " trying default MODIS names: only %s"\
                          %(str(contents)))
        names = def_names
        alt_names = def_alt_names
        for i in xrange(len(np.atleast_1d(contents))):
            if contents[i] in names:
                datasets.append(i)
        if not len(np.atleast_1d(datasets)):
            self.error_msg = "None of requested datasets %s found in %s"\
                %(str(names),filename) + ' ' + \
                "... trying default MODIS names: only %s"\
                %(str(contents))
            self.error = True
            if 'logger' in self or 'logger' in self.dict():
                self.logger.error(self.error_msg)
            return 0,(self.error,self.error_msg)
    trans_names = {}
    for (i,j) in enumerate(alt_names):
        trans_names[names[i]] = j
#trans_names = {names[i]:j for (i,j) in enumerate(alt_names)}
    alt_name = []
    this_name = []
    for i in datasets:
        this_name.append(contents[i])
        alt_name.append(trans_names[contents[i]])
    
    # Translate  some old stylies...
    trans = {'raa':'vaa','doys':'time'}
    for i in trans:
        if i in contents:
            ncontents[np.where(contents==i)[0]]=trans[i]
    # as a minimum, there needs to be some definition of one of
    # the terms in location

    # check how many dimensions this has
    # now find a dataset
    try:
        # This could be more general, but this will do for now as its useful
        # for spatial datasets
        QA_OK = np.array(\
                         [8, 72, 136, 200, 1032, 1288, 2056,2120, 2184, 2248])
        doy = f['doys'] - 2004000
        qa = f['qa']
        vza = f['vza']
        sza = f['sza']
        raa = f['raa']
        y = []
        for i in this_name:
            y.append(f[i])
        #mask = np.logical_or.reduce([qa==x for x in QA_OK ])
        if 'logger' in self or 'logger' in self.dict():
            self.logger.info(\
                         "sucessfully interpreted NpzFile dataset from %s"\
                             %filename)
            self.logger.info("sub-setting ...")
        controls = []
        locations = []
        grid = []
        qlocations = []
        thisshape = vza.shape
        starter = {'time':np.min(doy),'row':0,'col':0}
        delta = {'time':1,'row':1,'col':1}
        if len(np.atleast_1d(limits)) <3:
            from eoldas_Lib import set_default_limits
            old_loc = location
            location = np.array(['time','row','col'])
            lim2 = set_default_limits(location)
            for i in xrange(len(np.atleast_1d(limits))):
                ww = np.where(old_loc[i] == location)[0]
                lim2[ww] = list(limits[i])
            limits = lim2
        for i in xrange(len(np.atleast_1d(limits))):
            if limits[i][0] == None:
                limits[i][0] = starter[location[i]]
            if limits[i][1] == None:
                limits[i][1] = (thisshape[i]-1) + starter[location[i]]
            if limits[i][2] == None:
                limits[i][2]= delta[location[i]]
        limits = np.array(limits)
        start_doy = limits[0][0]
        end_doy =   limits[0][1]
        step_doy =  limits[0][2]
        start_row = limits[1][0]
        end_row =   limits[1][1]
        step_row =  limits[1][2]
        start_col = limits[2][0]
        end_col =   limits[2][1]
        step_col =  limits[2][2]
        gooddays = np.logical_and.reduce(np.concatenate(\
                            ([doy >= start_doy],[doy  <=end_doy])))
        qa = qa[gooddays,start_row:end_row+1,start_col:end_col+1]
        vza = vza[gooddays,start_row:end_row+1,start_col:end_col+1]*0.01
        sza = sza[gooddays,start_row:end_row+1,start_col:end_col+1]*0.01
        raa = raa[gooddays,start_row:end_row+1,start_col:end_col+1]*0.01
        yy = []
        for i in xrange(len(np.atleast_1d(this_name))):
            this = y[i]
            yy.append(this[gooddays,start_row:end_row+1,\
                           start_col:end_col+1]*0.0001)
        doy = doy[gooddays]
        # now do QA
        mask = np.zeros_like(qa).astype(bool)
        # loop over qa
        for j in xrange(len(np.atleast_1d(QA_OK))):
            ww = np.where(qa==QA_OK[j])
            mask[ww] = True
        # better look over data to check valid
        for j in xrange(len(np.atleast_1d(yy))):
            ww = np.where(yy[j] < 0)
            mask[ww] = False
        ww = np.where(mask)
        if 'logger' in self or 'logger' in self.dict():
            self.logger.debug('parsing dataset: %d samples look ok'\
                          %np.array(ww).shape[1]) 
        vza = vza[ww]
        sza = sza[ww]
        raa = raa[ww]
        doy= doy[ww[0]]
        row = ww[1]+start_row
        col = ww[2]+start_col
	locations  = np.array([doy,row,col])
	nnn = len(np.atleast_1d(locations[0]))
        orig = np.repeat(np.array([start_doy,start_row,start_col]),locations.shape[1]).reshape(locations.shape).T
        div = np.repeat(np.array([step_doy,step_row,step_col]),locations.shape[1]).reshape(locations.shape).T
        qlocations = ((locations.T - orig)/div.astype(float)).astype(int).T
        controls = np.array([np.ones_like(doy).astype(bool),\
                             vza,raa,sza,0*doy])
        y = []
        for i in xrange(len(np.atleast_1d(this_name))):
            this = yy[i]
            y.append(this[ww])
        grid = np.array(y)   
        fmt = 'BRDF-UCL'
        control = ['mask','vza','vaa','sza','saa']
        bands = alt_name
        if not np.array(grid).size:
            if 'logger' in self or 'logger' in self.dict():
                self.logger.error(\
                              "Warning: returning a zero-sized dataset ... "+\
                              " I wouldn;t try to do anything with it")
        # in case we dont have data for all bands
        mask =  np.logical_or.reduce([[this_name[i]==x for x in names] \
                                      for i in xrange(len(np.atleast_1d(this_name)))])
        sd = np.array('0.004 0.015 0.003 0.004 0.013 0.01 0.006'\
                      .split())[mask]
        sd = np.array([float(i) for i in sd.flatten()])\
            .reshape(sd.shape)
        nsamps = grid.shape[1]
        sd = sd.repeat(nsamps).reshape(grid.shape).T
        datasets = ParamStorage()
        datasets.data  = ParamStorage()
        datasets.name  = ParamStorage()
        datasets.name.fmt = fmt
        grid = grid.T
        datasets.data[name] = np.zeros([grid.shape[0],len(np.atleast_1d(names))])\
                                                        .astype(object)
        datasets.data[name][:,:] = None
        for i in xrange(len(np.atleast_1d(this_name))):
            ww = np.where(names == this_name[i])[0][0]
            datasets.data[name][:,ww] = grid[:,i]
        datasets.data.location = np.array(locations).T
        datasets.data.control = np.array(controls).T
        datasets.data.qlocation = np.array(qlocations).T
        datasets.name[name] = np.array(names)
        datasets.name.location = np.array(['time','row','col'])
        datasets.name.control = np.array(control)
        datasets.name.qlocation = limits
        datasets.name.bands = np.array(bands)
        datasets.data.sd = np.zeros([grid.shape[0],len(np.atleast_1d(names))])\
                                                        .astype(object)
        # for i in xrange(grid.shape[0]):
        # datasets.data.sd[i,:] = self.options.sd
        datasets.data.sd[:,:] = None
        for i in xrange(len(np.atleast_1d(this_name))):
            ww = np.where(names == this_name[i])[0][0]
            datasets.data.sd[:,ww] = sd[:,i]
        datasets.name.sd = np.array(names)
        if 'logger' in self or 'logger' in self.dict():
            self.logger.debug('finished parsing dataset')
    except:
        self.error_msg=\
            "a problem processing information from  %s as a NpzFile"\
                %filename
        self.error=True
        if 'logger' in self or 'logger' in self.dict():
            self.logger.info(self.error_msg)
        return 0,(self.error,self.error_msg)
    f.close()
    if 'logger' in self or 'logger' in self.dict():
        self.logger.info('... done')
    self.error=False
    self.error_msg=""
    return datasets,(self.error,self.error_msg)
Exemplo n.º 28
0
    def parseLoader(self, loaders):
        """
        Utility to load a set of terms from the list loaders 
        into the ParamStorage general

        If there are 3 terms in each loaders element, they refer to:
        1. name
        2. default value
        3. helper text
        If there is a fourth, it is associated with extras 
        (short parser option)
        """
        general = ParamStorage()
        general.__default__ = ParamStorage()
        general.__extras__ = ParamStorage()
        general.__helper__ = ParamStorage()

        for this in loaders:
            if len(this) > 1:
                general[this[0]] = this[1]
                general.__default__[this[0]] = this[1]
            else:
                general[this[0]] = None
                general.__default__[this[0]] = None
            if len(this) > 2:
                general.__helper__[this[0]] = this[2]
            else:
                general.__helper__[this[0]] = optparse.SUPPRESS_HELP
            if len(this) > 3:
                general.__extras__[this[0]] = "%s" % this[3]
            else:
                general.__extras__[this[0]] = None
            # make sure arrays arent numpy.ndarray
            if type(general.__default__[this[0]]) == np.ndarray:
                general.__default__[this[0]] = \
                        list(general.__default__[this[0]])

        self.top.update(self.__unload(general), combine=True)
Exemplo n.º 29
0
    def __init__(self,
                 confs,
                 logger=None,
                 logfile=None,
                 thisname=None,
                 name=None,
                 datadir=None,
                 logdir=None):
        '''
        Initialise the solver.
            
        This does the following:
            1. Read configuration file(s)
            2. Load operators
            3. Test the call to the cost function
            
        There can be multiple groups of configuration files, so
        self.confs, that holds the core information setup here
        can contain multiple configurations.
        
        The number of configurations is len(confs.infos) and
        the ith configuration is conf = self.confs.infos[i].
            
        Various loggers are available throughout the classes
        used, but the top level logger is self.confs.logger,
        so you can log with e.g.
            
        self.confs.logger.info('this is some info')
            
        The root operator is stored in self.confs.root[i]
        for the ith configuration, so the basic call to the cost
        function is:
            
        J,J_prime = self.confs[i].parameter.cost(None)
            
        '''
        from eoldas_ConfFile import ConfFile
        from eoldas_Lib import sortopt
        name = name or thisname
        if name == None:
            import time
            thistime = str(time.time())
            name = type(self).__name__
            name = "%s.%s" % (name, thistime)
        self.thisname = name

        self.confs = confs
        self.top = sortopt(self, 'top', ParamStorage())
        self.top.general = sortopt(self.top, 'general', ParamStorage())
        thisname = sortopt(self.top.general, 'name', thisname or self.thisname)
        logfile = sortopt(self.top.general, 'logfile', logfile or 'log.dat')
        logdir = sortopt(self.top.general, 'logdir', logfile or 'logs')
        datadir = sortopt(self.top.general, 'datadir', datadir or ['.'])
        self.logger = sortlog(self,logfile,logger or self.confs.logger,\
   name=self.thisname,logdir=logdir,debug=True)
        n_configs = len(self.confs.infos)
        self.confs.root = []
        self.have_unc = False
        try:
            logdir = logdir
        except:
            logdir = self.top.general.logdir

        # first set up parameter
        conf = confs.infos[0]

        general = conf.general
        op = conf.parameter
        if not 'parameter' in conf.dict():
            raise Exception('No parameter field found in %s item %d'%\
                                (conf.__doc__,0))
        general.is_spectral = sortopt(general, 'is_spectral', True)
        if not general.is_spectral:
            sort_non_spectral_model(op, conf.operator, logger=confs.logger)
        general.init_test = sortopt(general, 'init_test', False)
        confs.logger.info('loading parameter state')
        conf.parameter.name = 'Operator'
        parameter = eval(op.name)(op,general,\
                                      parameter=None,\
                                      logger=confs.logger,\
                                      name=name+".parameter",\
                                      datatype=list(conf.parameter.datatypes),\
                                      logdir=logdir,\
                                      logfile=logfile,\
                                      datadir=datadir)
        try:
            parameter.transform = parameter.options.x.transform
            parameter.invtransform = parameter.options.x.invtransform
        except:
            parameter.transform = parameter.options.x.names
            parameter.invtransform = parameter.options.x.names

        # we now have access to parameter.x.state, parameter.x.sd etc
        # and possibly parameter.y.state etc.
        operators = []
        for (opname, op) in conf.operator.dict().iteritems():
            if opname != 'helper':
                #pdb.set_trace()
                exec('from eoldas_%s import %s' % (op.name, op.name))
                # make sure the data limits and x bounds are the same
                # ... inherit from parameter
                op.limits = parameter.options.limits
                if not 'datatypes' in op.dict():
                    op.datatypes = 'x'
                thisop = eval(op.name)(op,general,parameter=parameter,\
                                           logger=confs.logger,\
                                           name=name+".%s-%s"%(thisname,opname),\
                                           datatype=list(op.datatypes),\
                                           logdir=logdir,\
                                           logfile=logfile,\
                                           datadir=datadir)
                # load from parameter
                thisop.loader(parameter)
                operators.append(thisop)
                try:
                    thisop.transform = parameter.options.x.transform
                    thisop.invtransform = parameter.options.x.invtransform
                except:
                    thisop.transform = parameter.options.x.names
                    thisop.invtransform = parameter.options.x.names
                thisop.ploaderMask = np.in1d(parameter.x_meta.state,
                                             thisop.x_meta.state)
                try:
                    thisop.invtransform = np.array(
                        thisop.invtransform[thisop.ploaderMask])
                    thisop.transform = np.array(
                        thisop.transform[thisop.ploaderMask])
                except:
                    ww = thisop.ploaderMask
                    thisop.invtransform = np.array(thisop.transform)[ww]
                    thisop.transform = np.array(thisop.transform)[ww]
        # sort the loaders
        parameter.operators = operators
        self.confs.root.append(parameter)
        # Now we have set up the operators
        # try out the cost function
        if general.init_test:
            self.logger.info('testing cost function calls')
            J, J_prime = self.confs.root[0].cost()
            self.logger.info('done')
        self.confs.root = np.array(self.confs.root)
Exemplo n.º 30
0
def demonstration():
    # set state to a filename
    # and it will be loaded with the data
    x = DemoClass()
    
    data = {'state':np.ones(2)*5.  ,'foo':np.ones(10)}
    name = {'state':'of the nation','foo':'bar'}
    this = {'data':data,'name':name}
    x.state = this
    print 1,x.state,x['state']

    
    x.oats = 'beans and barley-o'
    # nothing set so far
    print 2,x['state']
    # should return the same
    print 3,x.state
    x.state = 'test/data_type/input/test.brf'
    print 4,x.state
    print 5,x.Name.fmt

    # set state to a dict and
    # it will load from that
    data = {'state':np.zeros(10)}
    name = {'state':'foo'}
    x.state = {'data':data,'name':name}
    print 6,x.state
 
    # set from a ParamStorage
    # and it will be loaded
    this = ParamStorage()
    this.data = ParamStorage()
    this.name = ParamStorage()
    this.data.state = np.ones(10)
    this.name.state = 'bar'
    this.data.sd = np.ones(10)*2.
    this.name.sd = 'sd info'
    # assign the data
    x.state = this
    # access the data
    print 7,x.state
    # access another member
    # Data, Name == implicitly .state
    print 8,x.Data.sd
    print 9,x.Name.sd
    # set directly
    x.Name.sd = 'bar'
    print 10,x.Name.sd

    # set from a tuple (data,name)
    # or a list [data,name]
    data = 'foo'
    name = 'bar'
    x.state = (data,name)
    print 11,x.state
    x.state = [name,data]
    print 12,x.state

    # set from a numpy array
    x.state = np.array(np.arange(10))
    print 13,x.state

    # set from another state
    y = DemoClass()
    y.state = x.state
    x.state = x.state * 2
    print 'x state',x.state
    print 'y state',y.state

    # set from a float
    x.state = 100.
    print 14,x.state

    # another interesting feature
    # we have 2 special terms in demonstration
    # state and other
    # if we set up some strcture for data 
    # for other
    this = ParamStorage()
    this.data = ParamStorage()
    this.name = ParamStorage()
    this.data.other = np.ones(10)
    this.name.other = 'bar'
    # and the assign it to state
    x.state = this
    print 15,'state',x.state
    # we see state is unchanged
    # but other is also not set.
    print 16,'other',x.other
    # we load into other using:
    x.other = this
    print 'other',x.other
    # but if you look at the information contained
    print 17,x._other.to_dict()
    print 18,x._state.to_dict()
    
    # or better writtem as:
    print 19,x.var('state').to_dict()
    
    # you will see that state contains the other data that was loaded

    # a simple way to write out the data is to a pickle
    # x.write_pickle('xstate','x_state.pkl')
    # but try to avoid using the underscores
    print 20,"x state in pickle:",x.state
    SpecialVariable.write(x._state,'x_state.pkl',fmt='pickle')
    
    # which we can reload:  
    z = DemoClass()
    z.state = 'x_state.pkl'
    print 21,"z state read from pickle",z.state

    # which is the same as a forced read ...
    zz = DemoClass()
    zz.state = 'x_state.pkl'
    print 22,zz.state

    # read a brf file
    zz.Name.qlocation = [[170,365,1],[0,500,1],[200,200,1]]
    zz.state = 'test/data_type/input/interpolated_data.npz'
    print zz.state
    SpecialVariable.write(zz._state,'test/data_type/output/interpolated_data.pkl',fmt='pickle')
    zz.state = 'test/data_type/input/test.brf'
    print zz.state
Exemplo n.º 31
0
    def init(self,info=[],name=None,readers=[],log_terms={},\
                                                datadir=None,env=None,\
                                                header=None,writers={},\
                                                simple=False,logger=None):
        '''
        Initialise information in a SpecialVariable instance.
            
        Options:
            info        :   Information tthat can be passed through to reader
                            methods (a list).
            thisname    :   a name to use to identify this instance in any
                            logging. By default this is None. If thisname is set
                            to True, then logging is to stdout.
            readers     :   A list of reader methods that are pre-pended to 
                            those already contained in the class.
            log_terms   :   A dictionary of log options. By default
                            {'logfile':None,'logdir':'log','debug':True}
                            If thisname is set, and logfile specified, then 
                            logs are logged to that file. If thisname is set
                            to True, then logging is to stdout.
            datadir     :   A list of directories to search for data files
                            to interpret if the SpecialVariable is set to a 
                            string.
            env         :   An environment variable that can be used to extend
                            the datadir variable.
            header      :   A header string to use to identify pickle files.
                            By default, this is set to
                            "EOLDAS -- plewis -- UCL -- V0.1"
            simple      :   A flag to swicth off the 'complicated' 
                            interpretation methods, i.e. just set and return 
                            variables literally, do not try to interpret them.
            
        
        '''
        self.set('simple',True)
        if name == None:
           import time
           thistime = str(time.time())
           name = type(self).__name__
           name =  "%s.%s" % (name,thistime)
        self.thisname = name

        # this is where we will put any data     
        self.data = ParamStorage()
        self.name = ParamStorage()

        self.info = info

        self.datadir = datadir or ['.']
        self.env = env     

        init_read_write(self,header,readers,writers)
        
        # sort logging and log if thisname != None
        self.log_terms = {'logfile':None,'logdir':'log','debug':True}
        # override logging info
        for (key,value) in log_terms.iteritems():
            self.log_terms[key] = value
        self.logger= sortlog(self,self.log_terms['logfile'],logger,name=self.thisname,\
				logdir=self.log_terms['logdir'],\
				debug=self.log_terms['debug'])
        self.simple = simple
Exemplo n.º 32
0
class Parser():
    def __init__(self,args,name=None,general=None,log=False,logger=None,outdir=".",getopdir=False,parse=True):
        """
        Initialise parser class.

        This sets up the class defaults.
        
        Options:
            
        general=general: this over-rides and defaults with values 
                        set in parser general can be of the form:
                         1. class ParamStorage (i.e. the same form 
                            as self.general)
                         2. a command line list (where the first 
                            item in the list is ignored
                         3. a string containing a set of command line general
            
                         See self.parse() for more details on general 
                            2 and 3 as these simply make a call to tha
                            method.
            
        log=True         If log is set to True, then logging starts 
                            when this class is instanced.
                            Note that the logfile and logdir might 
                            change if subsequent calls to Parser.parse()
                            are made
        """
        if type(args) == str:
            args = args.split()
        self.dolog = log 
        self.log = log
        self.name = args[0]
        self.args = args[1:]
        self.fullargs = args
        self.store_fullargs = args
        if name == None:
           import time
           thistime = str(time.time())
           name = type(self).__name__
           name =  "%s.%s" % (name,thistime)
	self.thisname = name
        # find the following flags:
        # --conf | -c : conf
        # --datadir   : datadir
        datadir = [".","~/.eoldas",sys.path[0]+'/../bin',sys.path[0]+'/../confs',\
		sys.path[0]+'/../system_confs',sys.path[0]+'/../eoldaslib']
        conf = "default.conf"
        logfile = None
        logdir = "."
        self.top = ParamStorage ()
        self.top.general = ParamStorage ()
        self.top.general.__helper__ = ParamStorage ()
        self.top.general.__default__ = ParamStorage ()
        self.top.general.__extras__ = ParamStorage ()
        self.top.general.conf = []
        for i in xrange(len(self.args)):
            theseargs = self.args[i].split('=')
            if theseargs[0] == "--conf":
                conf = theseargs[1]
                self.top.general.conf.append(conf)
            elif theseargs[0][0:2] == "-c":
                if len(theseargs) > 2:
                    conf = theseargs[0][2:]
                else:
                    conf = self.args[i+1]
		self.top.general.conf.append(conf)
            elif theseargs[0] == "--datadir":
                datadir1 = theseargs[1].replace('[','').\
                                            replace(']','').split()
                [datadir1.append(datadir[i]) for i in \
                                            xrange(len(datadir))]
                datadir = datadir1
            elif theseargs[0] == "--logfile":
                logfile= theseargs[1]
            elif theseargs[0] == "--logdir":
                logdir = theseargs[1]
            elif theseargs[0] == "--outdir":
                outdir = theseargs[1]
        if self.top.general.conf == []:
	    self.top.general.conf = conf 
	if logfile == None:
            logfile = conf.replace('conf','log')
        self.top.general.here = os.getcwd()
        self.top.general.datadir = datadir
        self.top.general.logfile = logfile
        self.top.general.logdir = logdir
        self.top.general.outdir = outdir
        # add here to datadir
        # in addition to '.' to take account of the change of directory
        self.top.general.datadir = self.__add_here_to_datadir(\
                    self.top.general.here,self.top.general.datadir)
        self.top.general.datadir = self.__add_here_to_datadir(\
                    self.top.general.outdir,self.top.general.datadir)
        # cd to where the output is to be 
        self.__cd(self.top.general.outdir)
        # set up the default command line options
        self.default_loader() 
        # update with anything passed here
        if general and type(general) == ParamStorage: self.top.update(\
                    self.__unload(general),combine=True)
        # read the conf files to get any cmd line options
        self.logger = sortlog(self,self.top.general.logfile,logger,name=self.thisname,\
                    logdir=self.top.general.logdir)
        self.config = ConfFile(self.top.general.conf,name=self.thisname+'.config',\
		    loaders=self.loaders,datadir=self.top.\
                    general.datadir,logger=self.logger,logdir=self.top.general.logdir,\
                    logfile=self.top.general.logfile)   
        if len(self.config.configs) == 0:
            this = "Warning: Nothing doing ... you haven't set any configuration",\
						self.config.storelog
	    try:
                self.logger(this)
	    except:
	 	print "Called with args:"
		print "eoldas",self.args
	 	pass
            raise Exception(this)
            
        # now loaders contains all of the defaults set here
        # plus those from the config (config opver-rides defaults here)
        self.loaders = self.config.loaders
        # now convert loaders into parser information
        self.parseLoader(self.loaders)
        self.parse(self.fullargs)
        if general and type(general) == ParamStorage: 
		self.top.update(self.__unload(general),combine=True)
        if general and type(general) == str: 
		self.parse(general.split())
        if general and type(general) == list: 
		self.parse(general)
        # now update the info in self.config
        for i in self.config.infos:
            i.update(self.top,combine=True)
            # so now all terms in self.config.infos
            # contain information from the config file, updated by 
            # the cmd line
            i.logger = self.logger
            i.log()
        # move the information up a level
        self.infos = self.config.infos
        self.configs = self.config.configs
        self.config_log = self.config.storelog

        #if getopdir:
        #    self.sortnames()
        self.config.loglist(self.top)

        #del(self.config.infos)
        #del(self.config.configs)
        #del(self.config)
        self.__cd(self.top.general.here)

    def __add_here_to_datadir(self,here,datadir):
        from os import sep,curdir,pardir
        if type(datadir) == str:
            datadir = [datadir]
        iadd = 0
        for i in xrange(len(datadir)):
            j = i + iadd
            if datadir[j] == curdir or datadir[j] == pardir:
                tmp = datadir[:j]
                rest = datadir[j+1:]
                tmp.append("%s%s%s" % (here,sep,datadir[j]))
                tmp.append(datadir[j])
                iadd += 1
                for k in xrange(len(rest)):
                    tmp.append(rest[k])
                datadir = tmp
        return datadir       

    def default_loader(self):
        """
        Load up parser information for first pass
        """
        self.loaders = []
        self.top.general.here = os.getcwd()
        self.loaders.append(["datadir",['.',self.top.general.here],\
            "Specify where the data and or conf files are"])
        self.loaders.append(["passer",False,\
        "Pass over optimisation (i.e. report and plot the initial values)"])
        self.loaders.append(["outdir",None,\
        "Explicitly mspecify the results and processing output directory"])
        self.loaders.append(["verbose",False,"Switch ON verbose mode","v"])
        self.loaders.append(["debug",False,optparse.SUPPRESS_HELP,"d"])
        self.loaders.append(["conf","default.conf",\
        "Specify configuration file. Set multiple files by using the flag multiple times.","c"])
        self.loaders.append(["logdir","logs",\
        "Subdirectory to put log file in"])
        self.loaders.append(["logfile","logfile.logs","Log file name"])


    def parseLoader(self,loaders):
        """
        Utility to load a set of terms from the list loaders 
        into the ParamStorage general

        If there are 3 terms in each loaders element, they refer to:
        1. name
        2. default value
        3. helper text
        If there is a fourth, it is associated with extras 
        (short parser option)
        """
        general = ParamStorage ()
        general.__default__ = ParamStorage ()
        general.__extras__ = ParamStorage ()
        general.__helper__ = ParamStorage ()

        for this in loaders:
            if len(this) > 1:
                general[this[0]] = this[1]
                general.__default__[this[0]] = this[1]
            else:
                general[this[0]] = None
                general.__default__[this[0]] = None
            if len(this) > 2:
                general.__helper__[this[0]] = this[2]
            else:
                general.__helper__[this[0]] = optparse.SUPPRESS_HELP
            if len(this) > 3:
                general.__extras__[this[0]] = "%s" % this[3]
            else:
                general.__extras__[this[0]] = None
            # make sure arrays arent numpy.ndarray
            if type(general.__default__[this[0]]) == np.ndarray:
                general.__default__[this[0]] = \
                        list(general.__default__[this[0]])

        self.top.update(self.__unload(general),combine=True)


    def __list_to_string__(self,thisstr):
        """
        Utility to convert a list to some useable string
        """
        return(str(thisstr).replace('[','_').strip("']").\
            replace('.dat','').replace("_'","_").replace(",","").\
            replace(" ","").replace("''","_").replace("___","_").\
            replace("__","_"));


    def __cd(self,outdir):
        if not os.path.exists(outdir):
            try:
                os.makedirs(outdir)
            except OSerror:
                print "Fatal: Prevented from creating",outdir
                sys.exit(-1)
        try:
            os.chdir(outdir)
        except:
            print "Fatal: unable to cd to",outdir
            raise Exception("Fatal: unable to cd to %s"%outdir)

    def sortnames(self):
        """
        Utility code to sort out some useful filenames & directories
        """
        if self.top.general.outdir == None:
            basename = self.top.general.basename
            confnames =  self.__list_to_string__(self.top.general.conf)
            self.top.general.outdir = basename + "_conf_" + confnames
                
        self.__cd(self.top.general.outdir)
 
    def parse(self,args,log=False):
        '''
        Given a list such as sys.argv (of that form)
        or an equivalent string parse the general and store 
        in self.parser
        '''
        self.dolog = log or self.dolog
        if type(args) == type(""):
            args = args.split()
        args = args[1:]
        self.top.general.cmdline = str(args)
        usage = "usage: %prog [general] arg1 arg2"
        parser = OptionParser(usage,version="%prog 0.1")

        # we go through items in self.top.general and set up
        # parser general for each
        for this in sorted(self.top.general.__helper__.__dict__.keys()):
            # sort out the action
            # based on type of the default
            default=self.top.general.__default__[this]
            action="store"
            thistype=type(default)
            if thistype == type(None):
                thistype = type("")

            argss = '--%s'%this 
            dest = "%s"%this
            helper = self.top.general.__helper__[this]
            if type(default) == type([]):
                # list, so append
                action="store"
            elif type(default) == type(True):
                action="store_true"
                typer = "string"
            if thistype != type([]) and thistype  != type(True):
                typer='%s' % str(thistype).split("'")[1]
                # has it got extras?
                if self.top.general.__extras__[this] != None:
                    parser.add_option('-%s'%self.top.general.__extras__[\
                        this].lower(),argss,type="string",action=action,\
                        help=helper,default=str(default))
                else:
                    parser.add_option(argss,action=action,help=helper,\
                        type="string",default=str(default))
            elif ( thistype != type(True)):
                if self.top.general.__extras__[this] != None:
                    parser.add_option('-%s'%self.top.general.__extras__[\
                        this].lower(),argss,type="string",action=action,\
                        help=helper,default=str(default))
                else:
                    parser.add_option(argss,action=action,help=helper,\
                        default=str(default))
            if thistype == type(True):
                if self.top.general.__extras__[this] != None:
                    parser.add_option('-%s'%self.top.general.__extras__[\
                        this].lower(),argss,dest=dest,action=action,\
                        help=helper,default=default)
                else:
                    parser.add_option(argss,action=action,help=helper,\
                        dest=dest,default=default)
                that = this.split('.')
                argss = '--'
                for i in xrange(len(that)-1):
                    argss = argss + "%s." % that[i]
                argss = argss + 'no_%s' % that[-1]
                helper='The opposite of --%s'%this
                action='store_false'
                typer='%s' % str(thistype).split("'")[1]
                # has it got extras?
                if self.top.general.__extras__[this] != None:
                    parser.add_option('-%s'%self.top.general.__extras__[\
                        this].capitalize(),argss,dest=dest,action=action,\
                        help=helper)
                else:
                    parser.add_option(argss,action=action,dest=dest,\
                        help=helper)


        # we have set all option types as str, so we need to interpret 
        # them in__unload
        (general, args) = parser.parse_args(args)
        #for data_file in args:
        #  general.data_file.append(data_file)
        #general.data_file = list(np.array(general.data_file).flatten())
        #general.brf= list(np.array(general.brf).flatten())
        # load these into self.general
        self.top.update(self.__unload(general.__dict__),combine=True)
        #self.sortnames()
        if self.dolog:
            self.log = set_up_logfile(self.top.general.logfile,\
                                      logdir=self.top.general.logdir)
            self.log_report()

    def __unload(self,options):
        from eoldas_ConfFile import array_type_convert
        this = ParamStorage()
        this.general = ParamStorage()
        for (k,v) in options.iteritems():
            ps = this
            that = k.split('.')
            if len(that) == 1:
                ps = this.general
            else:
                for i in xrange(len(that)-1):
                    if not hasattr(ps,that[i]):
                        ps[that[i]] = ParamStorage()
                    ps = ps[that[i]]
            # set the value v which needs to 
            # to be interpreted
            ps[that[-1]] = array_type_convert(self.top,v)
        return this
Exemplo n.º 33
0
class DemoClass(ParamStorage):
    ''' 
        A demonstration class using SpecialVariable
        
        The behaviour we desire is that a SpecialVariable
        acts like a ParamStorage (i.e. we can get or set 
        by attribute or item 
        
        e.g. 
        
        x.state = 3 and
        x['state'] = 3
        
        give the same result, and
        
        print x['state'] and
        print x.state 
        
        give the same result. This is easy enough to achieve 
        for all cases other than getting from x.state. It turns out
        that __getattr__ does not override the default method
        for state *if* state is set in the class instance.
        
        To get around that, we have to use a fake name (fakes here)
        and instead of storing state, we store _state. This makes daling with
        with all of the conditions a little more complicated and a little 
        slower, but it allows a much more consistent interface.
        
        At any time, a SpecialVariable can simply be over-written by using 
        assigning to its fake name.
        
        e.g.
        
        instance the class
        
            x = demonstration()
        
        set a non-special value 'cheese'
        
            x.foo = 'bar'
        
        we can use this as x.foo or x['foo']
        
            print x.foo,x['foo']
        
        which should give bar bar
        
        now use the SpecialVariable. There are many way to load this up, 
        but an easy one is via a dictionary.
        
            data = {'state':np.ones(2)*5.  ,'foo':np.ones(10)}
            name = {'state':'of the nation','foo':'bar'}
            this = {'data':data,'name':name}
            x.state = this
        
            print x.state,x['state']
        
        which gives [ 5.  5.] [ 5.  5.], so we get the same from either
        approach. Note that what is returned from the SpecialVariable is 
        only what is in this['data']['state'], and that is fully the intention
        of the SpecialVariable class. It can be loaded with rich information
        from a range of sources, but if you want a quick interpretation of
        the data (i.e. x.state) you only get what is in x.state, or more fully,
        
            x._state.data.state
        
        The other data that we passed to the SpecialVariable is as it was
        when read in, but relative to x._state, i.e. we have:
        
            x._state.name.foo
        
        which is bar.
        
        If you want to directly access the SpecialVariable, you can use:
        
            x.get(x.fakes['state'])
        
        which is the same as
        
            x._state or x[x.fakes['state']]
        
        It is not adviseable to directly use the underscore access as the fakes 
        lookup dictionary can be changed. It is best to always use 
        x.fakes['state']. Indeed, if you want to override the 'special' nature
        of a term such as 'state', you can simply remove their entry from the 
        table:
        
            old_dict = x.fakes.copy()
            del x.fakes['state']
        
        Now, if you type:
        
            print x.state
        
        You get a KeyError for state, so it would have been better to:
        
            x.fakes = old_dict.copy()
            del x.fakes['state']
            x['state'] = x[old_dict['state']]
            print x.state
        
        which should give [ 5.  5.], but the type of x.state will have
        changed from SpecialVariable to np.ndarray.
        
        If you want to convert the SpecialVariable back to a dictionary
        you can do:
        
            print x[x.fakes['state']].to_dict()
        
        or a little less verbosely:
        
            print x._state.to_dict()
        
        
        '''
    

    def __init__(self,info=[],thisname=None,readers=[],\
                 datadir=["."],\
                 env=None,\
                 header=None,\
                 logger=None,
                 log_terms={},simple=False):
        '''
        Class initialisation.
            
        Set up self.state and self.other as SpecialVariables
        and initialise them to None.
            
        '''
        
        self.set('fakes',{'state':'_state','other':'_other'})
        
        nSpecial = len(self.get('fakes'))
        for i in self.fakes:
            thatname = thisname and "%s.%s"%(thisname,i)
                
            self[i] = SpecialVariable(logger=logger,info=info,thisname=thatname,\
                                     readers=readers,datadir=datadir,\
                                     env=env,\
                                     header=header,\
                                     log_terms=log_terms,\
                                     simple=False)
            self[i] = None
    

    get = lambda self,this :ParamStorage.__getattr__(self,this)
    get.__name__ = 'get'
    get.__doc__ = '''
       An alternative interface to get the value of a class member
       that by-passes any more complex mechanisms. This returns the 'true'
       value of a class member, as opposed to an interpreted value.
    '''
    set = lambda self,this,that :ParamStorage.__setattr__(self,this,that)
    set.__name__ = 'set'
    set.__doc__ = '''
        An alternative interface to set the value of a class member
        that by-passes any more complex mechanisms. This sets the 'true'
        value of a class member, as opposed to an interpreted value.
        '''

    var = lambda self,this : self[self['fakes'][this]]
    var.__name__='var'
    var.__doc__ = '''
        Return the data associated with SpecialVariable this, 
        rather than an interpretation of it
        '''
    
    
    def __set_if_unset(self,name,value):
        '''
        A utility to check if the requested attribute
        is not currently set, and to set it if so.
        '''
        if name in self.fakes:
            fname = self.fakes[name]
            if not fname in self.__dict__:
                ParamStorage.__setattr__(self,fname,value)
                return True
        else:
            if not name in self.__dict__:  
                ParamStorage.__setattr__(self,name,value)
                return True
        return False

    
    def __getattr__(self,name):
        '''
            get attribute, e.g. return self.state
        '''
        return self.__getitem__(name)

    def __setattr__(self,name,value):
        '''
            set attribute, e.g. self.state = 3
        '''
        if not self.__set_if_unset(name,value):
            self.__setitem__(name,value,nocheck=True)
 
    def __getitem__(self,name):
        '''
            get item for class, e.g. x = self['state']
        '''
        if name in ['Data','Name']:
            return self._state[name.lower()]
        elif name in ['Control','Location']:
            return self._state[name.lower()]
        elif name in self.fakes:
            this = self.get(self.fakes[name])
            return SpecialVariable.__getitem__(this,name)
        else:
            this = self.get(name)
            return self.__dict__.__getitem__ ( name )

    def __setitem__(self,name,value,nocheck=False):
        '''
            set item for class e.g. self['state'] = 3
        '''
        if nocheck or not self.__set_if_unset(name,value):
            if name in ['Data','Name']:
                self._state[name.lower()] = value
            elif name in ['Control','Location']:
                self._state[name.lower()] = value
            elif name in self.fakes:
                this = self.get(self.fakes[name])
                SpecialVariable.__setattr__(this,name,value)
            else:
                this = self.get(name)
                ParamStorage.__setattr__(self,name,value)
Exemplo n.º 34
0
class Parser():
    def __init__(self,
                 args,
                 name=None,
                 general=None,
                 log=False,
                 logger=None,
                 outdir=".",
                 getopdir=False,
                 parse=True):
        """
        Initialise parser class.

        This sets up the class defaults.
        
        Options:
            
        general=general: this over-rides and defaults with values 
                        set in parser general can be of the form:
                         1. class ParamStorage (i.e. the same form 
                            as self.general)
                         2. a command line list (where the first 
                            item in the list is ignored
                         3. a string containing a set of command line general
            
                         See self.parse() for more details on general 
                            2 and 3 as these simply make a call to tha
                            method.
            
        log=True         If log is set to True, then logging starts 
                            when this class is instanced.
                            Note that the logfile and logdir might 
                            change if subsequent calls to Parser.parse()
                            are made
        """
        if type(args) == str:
            args = args.split()
        self.dolog = log
        self.log = log
        self.name = args[0]
        self.args = args[1:]
        self.fullargs = args
        self.store_fullargs = args
        if name == None:
            import time
            thistime = str(time.time())
            name = type(self).__name__
            name = "%s.%s" % (name, thistime)
        self.thisname = name
        # find the following flags:
        # --conf | -c : conf
        # --datadir   : datadir
        datadir = [".","~/.eoldas",sys.path[0]+'/../bin',sys.path[0]+'/../confs',\
  sys.path[0]+'/../system_confs',sys.path[0]+'/../eoldaslib']
        conf = "default.conf"
        logfile = None
        logdir = "."
        self.top = ParamStorage()
        self.top.general = ParamStorage()
        self.top.general.__helper__ = ParamStorage()
        self.top.general.__default__ = ParamStorage()
        self.top.general.__extras__ = ParamStorage()
        self.top.general.conf = []
        for i in xrange(len(self.args)):
            theseargs = self.args[i].split('=')
            if theseargs[0] == "--conf":
                conf = theseargs[1]
                self.top.general.conf.append(conf)
            elif theseargs[0][0:2] == "-c":
                if len(theseargs) > 2:
                    conf = theseargs[0][2:]
                else:
                    conf = self.args[i + 1]
                self.top.general.conf.append(conf)
            elif theseargs[0] == "--datadir":
                datadir1 = theseargs[1].replace('[','').\
                                            replace(']','').split()
                [datadir1.append(datadir[i]) for i in \
                                            xrange(len(datadir))]
                datadir = datadir1
            elif theseargs[0] == "--logfile":
                logfile = theseargs[1]
            elif theseargs[0] == "--logdir":
                logdir = theseargs[1]
            elif theseargs[0] == "--outdir":
                outdir = theseargs[1]
        if self.top.general.conf == []:
            self.top.general.conf = conf
        if logfile == None:
            logfile = conf.replace('conf', 'log')
        self.top.general.here = os.getcwd()
        self.top.general.datadir = datadir
        self.top.general.logfile = logfile
        self.top.general.logdir = logdir
        self.top.general.outdir = outdir
        # add here to datadir
        # in addition to '.' to take account of the change of directory
        self.top.general.datadir = self.__add_here_to_datadir(\
                    self.top.general.here,self.top.general.datadir)
        self.top.general.datadir = self.__add_here_to_datadir(\
                    self.top.general.outdir,self.top.general.datadir)
        # cd to where the output is to be
        self.__cd(self.top.general.outdir)
        # set up the default command line options
        self.default_loader()
        # update with anything passed here
        if general and type(general) == ParamStorage:            self.top.update(\
self.__unload(general),combine=True)
        # read the conf files to get any cmd line options
        self.logger = sortlog(self,self.top.general.logfile,logger,name=self.thisname,\
                    logdir=self.top.general.logdir)
        self.config = ConfFile(self.top.general.conf,name=self.thisname+'.config',\
      loaders=self.loaders,datadir=self.top.\
                    general.datadir,logger=self.logger,logdir=self.top.general.logdir,\
                    logfile=self.top.general.logfile)
        if len(self.config.configs) == 0:
            this = "Warning: Nothing doing ... you haven't set any configuration",\
      self.config.storelog
            try:
                self.logger(this)
            except:
                print "Called with args:"
                print "eoldas", self.args
                pass
            raise Exception(this)

        # now loaders contains all of the defaults set here
        # plus those from the config (config opver-rides defaults here)
        self.loaders = self.config.loaders
        # now convert loaders into parser information
        self.parseLoader(self.loaders)
        self.parse(self.fullargs)
        if general and type(general) == ParamStorage:
            self.top.update(self.__unload(general), combine=True)
        if general and type(general) == str:
            self.parse(general.split())
        if general and type(general) == list:
            self.parse(general)
        # now update the info in self.config
        for i in self.config.infos:
            i.update(self.top, combine=True)
            # so now all terms in self.config.infos
            # contain information from the config file, updated by
            # the cmd line
            i.logger = self.logger
            i.log()
        # move the information up a level
        self.infos = self.config.infos
        self.configs = self.config.configs
        self.config_log = self.config.storelog

        #if getopdir:
        #    self.sortnames()
        self.config.loglist(self.top)

        #del(self.config.infos)
        #del(self.config.configs)
        #del(self.config)
        self.__cd(self.top.general.here)

    def __add_here_to_datadir(self, here, datadir):
        from os import sep, curdir, pardir
        if type(datadir) == str:
            datadir = [datadir]
        iadd = 0
        for i in xrange(len(datadir)):
            j = i + iadd
            if datadir[j] == curdir or datadir[j] == pardir:
                tmp = datadir[:j]
                rest = datadir[j + 1:]
                tmp.append("%s%s%s" % (here, sep, datadir[j]))
                tmp.append(datadir[j])
                iadd += 1
                for k in xrange(len(rest)):
                    tmp.append(rest[k])
                datadir = tmp
        return datadir

    def default_loader(self):
        """
        Load up parser information for first pass
        """
        self.loaders = []
        self.top.general.here = os.getcwd()
        self.loaders.append(["datadir",['.',self.top.general.here],\
            "Specify where the data and or conf files are"])
        self.loaders.append(["passer",False,\
        "Pass over optimisation (i.e. report and plot the initial values)"])
        self.loaders.append(["outdir",None,\
        "Explicitly mspecify the results and processing output directory"])
        self.loaders.append(["verbose", False, "Switch ON verbose mode", "v"])
        self.loaders.append(["debug", False, optparse.SUPPRESS_HELP, "d"])
        self.loaders.append(["conf","default.conf",\
        "Specify configuration file. Set multiple files by using the flag multiple times.","c"])
        self.loaders.append(["logdir","logs",\
        "Subdirectory to put log file in"])
        self.loaders.append(["logfile", "logfile.logs", "Log file name"])

    def parseLoader(self, loaders):
        """
        Utility to load a set of terms from the list loaders 
        into the ParamStorage general

        If there are 3 terms in each loaders element, they refer to:
        1. name
        2. default value
        3. helper text
        If there is a fourth, it is associated with extras 
        (short parser option)
        """
        general = ParamStorage()
        general.__default__ = ParamStorage()
        general.__extras__ = ParamStorage()
        general.__helper__ = ParamStorage()

        for this in loaders:
            if len(this) > 1:
                general[this[0]] = this[1]
                general.__default__[this[0]] = this[1]
            else:
                general[this[0]] = None
                general.__default__[this[0]] = None
            if len(this) > 2:
                general.__helper__[this[0]] = this[2]
            else:
                general.__helper__[this[0]] = optparse.SUPPRESS_HELP
            if len(this) > 3:
                general.__extras__[this[0]] = "%s" % this[3]
            else:
                general.__extras__[this[0]] = None
            # make sure arrays arent numpy.ndarray
            if type(general.__default__[this[0]]) == np.ndarray:
                general.__default__[this[0]] = \
                        list(general.__default__[this[0]])

        self.top.update(self.__unload(general), combine=True)

    def __list_to_string__(self, thisstr):
        """
        Utility to convert a list to some useable string
        """
        return(str(thisstr).replace('[','_').strip("']").\
            replace('.dat','').replace("_'","_").replace(",","").\
            replace(" ","").replace("''","_").replace("___","_").\
            replace("__","_"))

    def __cd(self, outdir):
        if not os.path.exists(outdir):
            try:
                os.makedirs(outdir)
            except OSerror:
                print "Fatal: Prevented from creating", outdir
                sys.exit(-1)
        try:
            os.chdir(outdir)
        except:
            print "Fatal: unable to cd to", outdir
            raise Exception("Fatal: unable to cd to %s" % outdir)

    def sortnames(self):
        """
        Utility code to sort out some useful filenames & directories
        """
        if self.top.general.outdir == None:
            basename = self.top.general.basename
            confnames = self.__list_to_string__(self.top.general.conf)
            self.top.general.outdir = basename + "_conf_" + confnames

        self.__cd(self.top.general.outdir)

    def parse(self, args, log=False):
        '''
        Given a list such as sys.argv (of that form)
        or an equivalent string parse the general and store 
        in self.parser
        '''
        self.dolog = log or self.dolog
        if type(args) == type(""):
            args = args.split()
        args = args[1:]
        self.top.general.cmdline = str(args)
        usage = "usage: %prog [general] arg1 arg2"
        parser = OptionParser(usage, version="%prog 0.1")

        # we go through items in self.top.general and set up
        # parser general for each
        for this in sorted(self.top.general.__helper__.__dict__.keys()):
            # sort out the action
            # based on type of the default
            default = self.top.general.__default__[this]
            action = "store"
            thistype = type(default)
            if thistype == type(None):
                thistype = type("")

            argss = '--%s' % this
            dest = "%s" % this
            helper = self.top.general.__helper__[this]
            if type(default) == type([]):
                # list, so append
                action = "store"
            elif type(default) == type(True):
                action = "store_true"
                typer = "string"
            if thistype != type([]) and thistype != type(True):
                typer = '%s' % str(thistype).split("'")[1]
                # has it got extras?
                if self.top.general.__extras__[this] != None:
                    parser.add_option('-%s'%self.top.general.__extras__[\
                        this].lower(),argss,type="string",action=action,\
                        help=helper,default=str(default))
                else:
                    parser.add_option(argss,action=action,help=helper,\
                        type="string",default=str(default))
            elif (thistype != type(True)):
                if self.top.general.__extras__[this] != None:
                    parser.add_option('-%s'%self.top.general.__extras__[\
                        this].lower(),argss,type="string",action=action,\
                        help=helper,default=str(default))
                else:
                    parser.add_option(argss,action=action,help=helper,\
                        default=str(default))
            if thistype == type(True):
                if self.top.general.__extras__[this] != None:
                    parser.add_option('-%s'%self.top.general.__extras__[\
                        this].lower(),argss,dest=dest,action=action,\
                        help=helper,default=default)
                else:
                    parser.add_option(argss,action=action,help=helper,\
                        dest=dest,default=default)
                that = this.split('.')
                argss = '--'
                for i in xrange(len(that) - 1):
                    argss = argss + "%s." % that[i]
                argss = argss + 'no_%s' % that[-1]
                helper = 'The opposite of --%s' % this
                action = 'store_false'
                typer = '%s' % str(thistype).split("'")[1]
                # has it got extras?
                if self.top.general.__extras__[this] != None:
                    parser.add_option('-%s'%self.top.general.__extras__[\
                        this].capitalize(),argss,dest=dest,action=action,\
                        help=helper)
                else:
                    parser.add_option(argss,action=action,dest=dest,\
                        help=helper)

        # we have set all option types as str, so we need to interpret
        # them in__unload
        (general, args) = parser.parse_args(args)
        #for data_file in args:
        #  general.data_file.append(data_file)
        #general.data_file = list(np.array(general.data_file).flatten())
        #general.brf= list(np.array(general.brf).flatten())
        # load these into self.general
        self.top.update(self.__unload(general.__dict__), combine=True)
        #self.sortnames()
        if self.dolog:
            self.log = set_up_logfile(self.top.general.logfile,\
                                      logdir=self.top.general.logdir)
            self.log_report()

    def __unload(self, options):
        from eoldas_ConfFile import array_type_convert
        this = ParamStorage()
        this.general = ParamStorage()
        for (k, v) in options.iteritems():
            ps = this
            that = k.split('.')
            if len(that) == 1:
                ps = this.general
            else:
                for i in xrange(len(that) - 1):
                    if not hasattr(ps, that[i]):
                        ps[that[i]] = ParamStorage()
                    ps = ps[that[i]]
            # set the value v which needs to
            # to be interpreted
            ps[that[-1]] = array_type_convert(self.top, v)
        return this
Exemplo n.º 35
0
class DModel_Operator(Operator):
    def preload_prepare(self):
        '''
        Here , we use preload_prepare to make sure 
        the x & any y data are gridded for this
        operator. This greatly simplifies the 
        application of the differential operator.
            
        This method is called before any data are loaded, 
        so ensures they are loaded as a grid.
        '''
        from eoldas_Lib import sortopt
        for i in np.array(self.options.datatypes).flatten():
            # mimic setting the apply_grid flag in options
            if self.dict().has_key('%s_state' % i):
                self['%s_state' % i].options[i].apply_grid = True
        self.novar = sortopt(self, 'novar', False)
        self.gamma_col = sortopt(self, 'gamma_col', None)
        self.beenHere = False

    def postload_prepare(self):
        '''
        This is called on initialisation, after data have been read in
           
	Here, we load parameters specifically associated with the
	model H(x).

	In the case of this differential operator, there are:

		model_order	: order of the differential operator
				  (integer)
		wraparound	: edge conditions
				  Can be:
					periodic
					none
					reflexive
		lag		: The (time/space) lag at which
				  the finite difference is calculated
				  in the differential operator here.
				  If this is 1, then we take the difference
				  between each sample point and its neighbour.
				  This is what we normally use. The main
				  purpose of this mechanism is to allow
				  differences at multiple lags to be 
			          calculated (fuller autocorrelation function
				  constraints as in kriging)

				  Multiple lags can be specified (which you could
				  use to perform kriging), in which case lag
				  weight should also be specified.

		lag_weight	: The weight associated with each lag. This will
				  generally be decreasing with increasing lag for
				  a 'usual' autocorrelation function. There is no point
				  specifying this if only a single lag is specified
				  as the function is normalised.
				
	If the conditions are specified as periodic
	the period of the function can also be specified, e.g. 
	for time varying data, you could specify 365 for the 
	periodic period.

	These are specified in the configuration file as

		operator.modelt.rt_model.model_order
		operator.modelt.rt_model.wraparound
		operator.modelt.rt_model.lag
		operator.modelt.rt_model.lag_weight

	The default values (set here) are 1, 'none', 1 and 1 respectively.

	To specify the period for `periodic` specify e.g.:

	[operator.modelt.rt_model]
	wraparound=periodic,365

	The default period is set to 0, which implies that it is periodic
	on whatever the data extent is.

	Or for multiple lags:

	[operator.modelt.rt_model]
	lag=1,2,3,4,5
	lag_weight=1,0.7,0.5,0.35,0.2

	NB this lag mechanism has not yet been fully tested
        and should be used with caution. It is intended more
	as a placeholder for future developments.

	Finally, we can also decide to work with 
	inverse gamma (i.e. an uncertainty-based measure)

	This is achieved by setting the flag

		operator.modelt.rt_model.inverse_gamma=True

	This flag should be set if you intend to estimate gamma in the
	Data Assimilation. Again, the is experimental and should be used
	with caution.
 
        '''
        from eoldas_Lib import sortopt
        self.rt_model = sortopt(self.options, 'rt_model', ParamStorage())
        self.rt_model.lag = sortopt(self.rt_model, 'lag', 1)
        self.rt_model.inverse_gamma= \
         sortopt(self.rt_model,'inverse_gamma',False)
        self.rt_model.model_order = \
   sortopt(self.rt_model,'model_order',1)
        self.rt_model.wraparound = \
          sortopt(self.rt_model,'wraparound','none')
        self.rt_model.wraparound_mod = 0
        if np.array(self.rt_model.wraparound).size == 2 and \
         np.array(self.rt_model.wraparound)[0] == 'periodic':
            self.rt_model.wraparound_mod = \
          np.array(self.rt_model.wraparound)[1]
            self.rt_model.wraparound = \
                               np.array(self.rt_model.wraparound)[0]
        self.rt_model.lag = \
   sortopt(self.rt_model,'lag',[1])
        self.rt_model.lag = np.array(self.rt_model.lag).flatten()

        self.rt_model.lag_weight = \
          sortopt(self.rt_model,'lag_weight',[1.]*\
          self.rt_model.lag.size)
        self.rt_model.lag_weight = np.array(\
          self.rt_model.lag_weight).flatten().astype(float)

        if self.rt_model.lag_weight.sum() == 0:
            self.rt_model.lag_weight[:] = np.ones(
                self.rt_model.lag_weight.size)

        self.rt_model.lag_weight = self.rt_model.lag_weight\
          / self.rt_model.lag_weight.sum()

    def setH(self):
        '''
            This method sets up the matrices required for the model.
            
            This operator is written so that it can apply smoothing
            in different dimensions. This is controlled by the model
            state vector. 
            
            The names of the states are stored in self.x_meta.location
            and the associated location information in self.x_meta.location.
            So, we look through these looking for matches, e.g. 'row' in 
            location and 'gamma_row' in names would mean that we want
            to apply the model over the row dimension. There should be only
            one gamma term in the state vectors for this operator. If you
            give more than one, only the last one will be used.
            
            NOT YET IMPLEMENTED: 
            The model can be applied to multiple dimensions by specifying
            e.g. gamma_time_row. If you want separate gammas for e.g. 
            time and row, then you should use separate operators. If
            gamma_roccol is specified, then the model applies to
            Euclidean distance in row/col space.
            
            Formally, the problem can be stated most simply as a matrix
            D so that gamma D x is the rate of change of x with respect 
            to the target location variable (time, row, col etc). 
            The job of this method then is to form and store D.
            
            The main complication to this is we have to split up x into
            those terms that we will apply D to (x2 here) and separately
            pull out the gamma terms. The resultant matrix D then needs to
            be re-formed so as to apply to the whole vector x, rather than 
            just x2. We do this with masks.
            
            On input, x is a 1D vector.

        '''
        x, Cx1, xshape, y, Cy1, yshape = self.getxy()
        # the names of the variables in x
        names = np.array(self.x_meta.state)
        # the names of the location information (e.g. time, row, col)
        location = self.x_meta.location
        self.logger.info('Setting up model matrices...')

        if self.x_meta.is_grid:
            try:
                self.x.location = self.x_state.ungridder.location
                self.x.qlocation = self.x_state.ungridder.qlocation
            except:
                raise Exception("You are trying to ungrid a dataset that wasn't gridded using State.regrid()" +\
                    " so the ungridder information is not available. Either load the data using State.grid " +\
                    " or set it up some other way or avoid calling this method with this type of data")

        # first, reshape x from its 1-D form to
        # have the same shape as self.x.state. We store
        # this shape as xshape.
        xshape = self.x.state.shape

        # we can't change the tuple directly, so need a
        # vector representation that we can manipulate
        # This is xshaper
        xshaper = np.array(xshape)

        # the data are assumed loaded into x

        # At this point, x2 is just a copy of the full input vector x
        # mask then is a mask of the same size as self.x_meta.state
        # by deafult, this mask is True. We will modify it to
        # take out bits we dont want later.
        x2 = x.reshape(xshape)
        mask = np.ones_like(x2).astype(bool)

        # We now need to recognise any gamma terms that might be in
        # the state vector. Candidates are 'gamma_%s'%(location)
        # e.g. gamma_time.

        # The number of dimensions of x can vary, depending on how many
        # loaction terms are used, so its a little tricky to
        # pull the information out.
        # We loop over the locations, indexed as i
        self.linear.datamask = np.ones(xshape[-1]).astype(bool)
        for i in xrange(len(location)):
            # and form the name of the candidate term in the variable 'this'
            this = 'gamma_%s' % location[i]
            ww = np.where(this == names)[0]
            # Then we see if it appears in the names of the state variables
            if len(ww):
                # form a mask so we dont apply the operator to gamma
                # terms. Note that *all* gamma terms are masked
                # even though we only actually use the last one we
                # come across.
                # we use [...,ww[0]] because the identifier for the
                # state is always in the final dimension.
                mask[..., ww[0]] = False
                # We store ww[0] as it will alllow us to access gamma
                # in subsequent calls in this same way. This is
                # self.linear.gamma_col
                self.linear.gamma_col = ww[0]
                # and is used as ...
                gammas = x2[..., self.linear.gamma_col]
                self.linear.datamask[self.linear.gamma_col] = False
                # We want to store an index into which of the
                # location vector terms we are dealing with here.
                # This is
                self.linear.gamma_loc = i
                # Once we apply the mask to get rid of the gamma columns
                # we need to keep track of the new shape for x2
                # This will be x2shape
                xshaper[-1] -= 1

        self.linear.x2shape = tuple(xshaper)
        self.linear.x2mask = mask.flatten()
        # so, apply the mask to take out the gamma columns
        x2 = x[self.linear.x2mask].reshape(self.linear.x2shape)

        # We next need access to the location information
        # for the selected dimension self.linear.gamma_loc.
        # If the data are gridded, we need to form the relevant information
        # Ungridded data we can access location directly as it is explicitly
        # stored. We store the location vector as 'locations'
        try:
            locshape = gammas.shape
        except:
            # If no gamma term is given, it is implicit that it is
            # the first dimension of location, but we have no data to mask
            self.linear.gamma_col = None
            self.linear.gamma_loc = 0
            locshape = (0)
            gammas = x2[..., 0] * 0. + 1.0
        #if self.x_meta.is_grid:
        # the locational variable of interest is self.linear.gamma_loc
        # the grid is dimensioned e.g. [t,r,c,p]
        # so we need e.g. locations which is of dimension
        # e.g. [t,r,c]
        #    locations = self.x.location
        # access the ungridded location data
        lim = self.x_meta.qlocation[self.linear.gamma_loc]
        nloc = lim[1] - lim[0] + 1
        locations = self.x.location[..., self.linear.gamma_loc]
        locshape = tuple(np.array(self.x.location.shape)[:-1])

        for (i, lag) in enumerate(self.rt_model.lag):
            wt = self.rt_model.lag_weight[i]

            slocations = wt*(np.roll(locations,lag,\
    axis=self.linear.gamma_loc) - locations).astype(float)
            slocations2 = (locations - np.roll(locations,-lag,\
    axis=self.linear.gamma_loc)).astype(float)

            # If there is no variation, it is a waste of time to calculate
            # the derivative
            if i == 0 and np.abs(slocations).sum() + np.abs(
                    slocations2).sum() == 0:
                # there is no variation here
                self.novar = True
                return 0
            self.novar = False
            ww = np.where(slocations > 0)
            mod = int(self.rt_model.wraparound_mod
                      ) / lim[-1] or slocations.shape[self.linear.gamma_loc]
            if self.rt_model.wraparound == 'reflexive':
                slocations[ww] = 0.
                #slocations[ww] = -np.fmod(mod - slocations[ww],mod)
            elif self.rt_model.wraparound == 'periodic':
                if self.rt_model.wraparound_mod == 0:
                    slocations[ww] = slocations2[ww]
                else:
                    slocations[ww] = -np.fmod(mod - slocations[ww], mod)
            else:  # none
                slocations[ww] = 0.
            ww = np.where(slocations != 0)
            slocations[ww] = 1. / slocations[ww]

            if i == 0:
                # Form the D matrix. This is of the size required to
                # process the x2 data, and this is the most convenient
                # form to use it in
                m = np.zeros(slocations.shape * 2)
            ww = np.where(slocations != 0)
            ww2 = np.array(ww).copy()
            ww2[self.linear.gamma_loc] = ww2[self.linear.gamma_loc] - lag
            ww2 = tuple(ww2)
            m[ww * 2] = m[ww * 2] - slocations[ww]
            if False and self.rt_model.wraparound == 'reflexive':
                ww2 = np.abs(ww - lag)
                # this is looped as there might be multiple elements with the
                # same index for the reflecxive case
                if m.ndim > 2:
                    raise Exception(
                        "Not yet implemented: Can't use reflexive mode for multi-dimensions yet"
                    )
                for (c, j) in enumerate(ww2):
                    m[j, ww[c]] = m[j, ww[c]] + slocations[ww[c]]
            else:
                ww = tuple(ww)
                ww2 = tuple(ww2)
                m[ww2 + ww] = m[ww2 + ww] + slocations[ww]
        # fix for edge conditions
        dd = m.copy()
        dd = dd.reshape(tuple([np.array(self.linear.x2shape[:-1]).prod()]) * 2)
        ddw = np.where(dd.diagonal() == 0)[0]
        for d in (ddw):
            ds = -dd[d, :].sum()
            dd[d, :] += dd[d, :]
            dd[d, d] = ds
        m = dd.reshape(m.shape)
        self.logger.info('Caching model matrices...')
        #
        if np.array(xshape).prod() == Cy1.size:
            self.linear.C1 = Cy1.reshape(xshape)[mask]\
                    .reshape( self.linear.x2shape )
        elif xshape[1] == Cy1.size:
            self.linear.C1 = np.tile(Cy1, xshape[0])[mask.flatten()].reshape(
                self.linear.x2shape)
        else:
            raise Exception("Can't deal with full covar matrix in DModel yet")
        nn = slocations.flatten().size
        m = m.reshape(nn, nn)
        self.linear.D1 = np.matrix(m).T
        for i in xrange(1, self.rt_model.model_order):
            m = np.matrix(self.linear.D1).T * m
        self.linear.D1 = m
        self.logger.info('... Done')
        return True

    def J(self):
        '''
        A slightly modified J as its efficient to 
        precalculate things for this model
       
	J = 0.5 * x.T D1.T gamma^2 D1 x
 
        '''
        x, Cx1, xshape, y, Cy1, yshape = self.getxy()
        self.Hsetup()
        if self.novar:
            return 0
        xshape = self.x.state.shape
        try:
            if self.linear.gamma_col != None:
                gamma = x.reshape(self.x.state.shape)\
                    [...,self.linear.gamma_col].flatten()
            else:
                # no gamma variable, so use 1.0
                gamma = x.reshape(self.x.state.shape)\
                    [...,0].flatten()*0.+1.
        except:
            self.logger.error(
                'gamma_col not set ... recovering and assuming no variation here'
            )
            self.linear.gamma_col = None
            gamma = x.reshape(self.x.state.shape)[..., 0].flatten() * 0. + 1.
            self.novar = True
            self.Hsetup()
            return 0

        x2 = x[self.linear.x2mask].reshape(self.linear.x2shape)
        J = 0.
        i = 0
        if self.rt_model.inverse_gamma:
            tgamma = 1. / gamma
        else:
            tgamma = gamma
        for count in xrange(self.x.state.shape[-1]):
            if count != self.linear.gamma_col:
                C1 = np.diag(self.linear.C1[...,i].\
                             reshape(self.linear.D1.shape[0]))
                x2a = x2[..., i].reshape(self.linear.D1.shape[0])
                xg = np.matrix(x2a * tgamma).T
                dxg = self.linear.D1.T * xg

                J += np.array(0.5 * dxg.T * C1 * dxg)[0][0]
                i += 1

    #print x[0],J
        return np.array(J).flatten()[0]

    def J_prime_prime(self):
        '''
	Calculation of J''

	We already have the differntial operator
	self.linear.D1 and self.gamma
	after we call self.J_prime()

	Here, J'' = D1.T gamma^2 D1

	J' is of shape (nobs,nstates)
	which is the same as the shape of x
	
	D1 is of shape (nobs,nobs)
	which needs to be expanded to 
	(nobs,nstates,nobs,nstates)

	'''
        x, Cx1, xshape, y, Cy1, yshape = self.getxy()
        J, J_prime = self.J_prime()
        xshape = self.x.state.shape

        if not 'linear' in self.dict():
            self.linear = ParamStorage()
        if not 'J_prime_prime' in self.linear.dict():
            self.linear.J_prime_prime = \
         np.zeros(xshape*2)
        else:
            self.linear.J_prime_prime[:] = 0
        # we need an indexing system in case of multiple
        # nobs columns
        x2a = np.diag(np.ones(self.linear.x2shape[:-1]).flatten())
        try:
            gamma = self.linear.gamma.flatten()
        except:
            if self.linear.gamma_col != None:
                gamma = x.reshape(self.x.state.shape)\
                    [...,self.linear.gamma_col].flatten()
            else:
                # no gamma variable, so use 1.0
                gamma = x.reshape(self.x.state.shape)\
                    [...,0].flatten()*0.+1.
            gamma = self.linear.gamma.flatten()
        if self.rt_model.inverse_gamma:
            tgamma = 1. / gamma
            dg = 2. / (gamma * gamma * gamma)
        else:
            tgamma = gamma
            dg = 1.0
        nshape = tuple([np.array(self.linear.x2shape[:-1]).prod()])
        D1 = np.matrix(self.linear.D1.reshape(nshape * 2))
        i = 0
        # so, e.g. we have xshape as (50, 100, 2)
        # because one of those columns refers to the gamma value
        # self.linear.gamma_col will typically be 0
        for count in xrange(xshape[-1]):
            if count != self.linear.gamma_col:
                # we only want to process the non gamma col
                C1 = np.diag(self.linear.C1[...,i].\
                             reshape(self.linear.D1.shape[0]))
                xg = np.matrix(x2a * tgamma * tgamma)
                dxg = D1 * xg
                deriv = np.array(dxg.T * C1 * D1)
                # so we have gamma^2 D^2 which is the Hessian
                # we just have to put it in the right place now
                # the technical issue is indexing an array of eg
                # (50, 100, 2, 50, 100, 2)
                # but it might have more or fewer dimensions
                nd = len(np.array(xshape)[:-1])
                nshape = tuple(np.array(xshape)[:-1])
                if nd == 1:
                    self.linear.J_prime_prime[:, count, :,
                                              count] = deriv.reshape(nshape *
                                                                     2)
                elif nd == 2:
                    self.linear.J_prime_prime[:, :, count, :, :,
                                              count] = deriv.reshape(nshape *
                                                                     2)
                elif nd == 3:
                    self.linear.J_prime_prime[:, :, :, count, :, :, :,
                                              count] = deriv.reshape(nshape *
                                                                     2)
                else:
                    self.logger.error(
                        "Can't calculate Hessian for %d dimensions ... I can only do up to 3"
                        % nd)

        #ww = np.where(deriv)
        #ww2 = tuple([ww[0]]) + tuple([ww[0]*0+count]) \
        #		+ tuple([ww[1]] )+ tuple([ww[0]*0+count])
        #x1 = deriv.shape[0]
        #x2 = self.linear.J_prime_prime.shape[-1]
        #xx = self.linear.J_prime_prime.copy()
        #xx = xx.reshape(x1,x2,x1,x2)
        #xx[ww2] = deriv[ww]
        #self.linear.J_prime_prime = xx.reshape(self.linear.J_prime_prime.shape)
                i += 1
        if self.linear.gamma_col != None:
            c = self.linear.gamma_col
            nd = len(np.array(xshape)[:-1])
            nshape = tuple(np.array(xshape)[:-1])
            deriv = np.diag(dg * 2 * J / (tgamma * tgamma)).reshape(nshape * 2)
            if nd == 1:
                self.linear.J_prime_prime[:, c, :, c] = deriv
            elif nd == 2:
                self.linear.J_prime_prime[:, :, c, :, :, c] = deriv
            elif nd == 3:
                self.linear.J_prime_prime[:, :, :, c, :, :, :, c] = deriv
            else:
                self.logger.error(
                    "Can't calculate Hessian for %d dimensions ... I can only do up to 3"
                    % nd)

        #dd = np.arange(nshape[0])
        #x1 = dd.shape[0]
        #x2 = self.linear.J_prime_prime.shape[-1]
        #xx = self.linear.J_prime_prime.copy()
        #xx = xx.reshape(x1,x2,x1,x2)
        #xx[dd,dd*0+self.linear.gamma_col,\
        #	dd,dd*0+self.linear.gamma_col] = dg*2*J/(tgamma*tgamma)
        #self.linear.J_prime_prime = xx.reshape(self.linear.J_prime_prime.shape)
        n = np.array(xshape).prod()
        return J, J_prime, self.linear.J_prime_prime.reshape(n, n)

    def J_prime(self):
        '''
            A slightly modified J as its efficient to 
            precalculate things for this model
           
	   J' = D.T gamma^2 D x 
            
        '''
        J = self.J()
        if self.novar:
            return 0, self.nowt
        x, Cx1, xshape, y, Cy1, yshape = self.getxy()
        x2 = x[self.linear.x2mask].reshape(self.linear.x2shape)
        if self.linear.gamma_col != None:
            gamma = x.reshape(self.x.state.shape)\
                [...,self.linear.gamma_col].flatten()
        else:
            # no gamma variable, so use 1.0
            gamma = x.reshape(self.x.state.shape)\
                [...,0].flatten()*0.+1.
        #gamma = self.linear.gamma.flatten()

        if self.rt_model.inverse_gamma:
            tgamma = 1. / gamma
            dg = -1. / (gamma * gamma)
        else:
            tgamma = gamma
            dg = 1.0

        g2 = tgamma * tgamma
        xshape = self.x.state.shape
        J_prime = np.zeros((x.shape[0] / xshape[-1], xshape[-1]))
        D2x_sum = 0.
        # loop over the non gamma variables
        i = 0
        # store gamma in case needed elsewhere
        self.linear.gamma = gamma
        for count in xrange(self.x.state.shape[-1]):
            if count != self.linear.gamma_col:
                C1 = np.diag(self.linear.C1[...,i].\
                             reshape(self.linear.D1.shape[0]))
                x2a = x2[..., i].reshape(self.linear.D1.shape[0])
                xg = np.matrix(x2a * tgamma).T
                dxg = self.linear.D1 * xg
                deriv = np.array(dxg.T * C1 * self.linear.D1)[0]
                J_prime[..., count] = deriv * tgamma
                #if self.linear.gamma_col != None:
                #        J_prime_gamma = deriv * x2a
                #        D2x_sum = D2x_sum + J_prime_gamma
                i += 1
        if self.linear.gamma_col != None:
            J_prime[..., self.linear.gamma_col] = dg * 2 * J / tgamma

        return J, J_prime

    def Hsetup(self):
        '''
        setup for the differential operator H(x) 
            
        '''
        if not self.beenHere and not 'H_prime' in self.linear.dict():
            self.logger.info('Setting up storage for efficient model operator')
            if 'y' in self.dict():
                self.linear.H = np.zeros(self.y.state.shape)
                self.linear.H_prime = np.zeros(self.y.state.shape * 2)
            else:
                self.linear.H = np.zeros(self.x.state.shape)
                self.linear.H_prime = np.zeros(self.x.state.shape * 2)
            self.setH()
            if self.novar:
                self.nowt = 0. * self.x.state
                #del self.linear.H_prime, self.linear.H
            self.beenHere = True
Exemplo n.º 36
0
    def scan_info(self,config,this,info,fullthis,fullinfo):
        """
        Take a ConfigParser instance config and scan info into 
        config.info. This is called recursively if needed.

        Parameters:
            config    : the configuration object
            this      : the current item to be parsed
            info      : where this item is to go
            fullthis  : the full name of this
            fullinfo  : the full (top level) version of info.
            
        """
        from eoldas_ConfFile import assoc_to_flat
        # find the keys in the top level
        # loop over 
        thiss = np.array(this.split('.'))
        # just in case .. is used as separator
        ww = np.where(thiss != '')
        thiss = thiss[ww]
        nextone = ''
        for i in xrange(1,len(thiss)-1):
            nextone = nextone + thiss[i] + '.'
        if len(thiss) > 1:
            nextone = nextone + thiss[-1]
        # first, check if its already there
        if not hasattr(info,thiss[0]):
            info[thiss[0]] = ParamStorage()
            info[thiss[0]].helper = []
        # load up the info
        if len(thiss) == 1:
            for option in config.options(fullthis):
                fulloption = option
                # option may have a '.' separated term as well
                options = np.array(option.split('.'))
                # tidy up any double dot stuff
                ww = np.where(options != '')
                options = options[ww]
                # need to iterate to make sure it is loaded 
                # at the right level
                # of the hierachy
                this_info = info[this]
                # so now this_info is at the base
                for i in xrange(len(options)-1):
                    if not hasattr(this_info,options[i]):
                        this_info[options[i]] = ParamStorage()
                        this_info[options[i]].helper = []
                    this_info = this_info[options[i]]
                option = options[-1]
                this_info[option] = array_type_convert(fullinfo,\
                                            config.get(fullthis,fulloption))
                if option[:6] == 'assoc_':
                    noption = option[6:]
                    this_info[noption] = assoc_to_flat(\
                                fullinfo.parameter.names,this_info[option],\
                                this_info[noption])
                    is_assoc = True
                else:
                    is_assoc = False
                if not hasattr(this_info,'helper'):
                    this_info.helper = []
                ndot = len(fullthis.split('.'))
                pres = ''
                for i in xrange(1,ndot):
                    pres += '  '
                if type(this_info.helper) == str:
                    this_info.helper += "\n%s%s.%-8s = %-8s" % \
                        (pres,fullthis,fulloption,str(this_info[option]))
                elif type(this_info.helper) == list:
                    this_info.helper.append("%s%s.%-8s = %-8s" % \
                    (pres,fullthis,fulloption,\
                    str(this_info[option])))
                if is_assoc:
                    if type(this_info.helper) == str:
                        this_info.helper += "\n%s%s.%-8s = %-8s" % \
                                (pres,fullthis,fulloption.replace\
                                ('assoc_',''),str(this_info[noption]))
                    elif type(this_info.helper) == list:
                        this_info.helper.append("%s%s.%-8s = %-8s" % \
                                (pres,fullthis,fulloption.replace\
                                 ('assoc_',''),str(this_info[noption])))
        else:
            self.scan_info(config,nextone,info[thiss[0]],fullthis,fullinfo)
            if thiss[-1][:6] == 'assoc_' and thiss[0] in fullinfo.dict():
                # only do this operation when at the top level
                noption = thiss[-1][6:]
                option = thiss[-1]
                this_info = info
                fulloption = thiss[0]
                this_info = this_info[thiss[0]]
                for i in xrange(1,len(thiss)-1):
                    this_info = this_info[thiss[i]]
                    fulloption = '%s.%s' % (fulloption,thiss[i])
                fulloption = '%s.%s' % (fulloption,noption)
                #this_info[noption] = assoc_to_flat(fullinfo.parameter.names\
                #                                   ,this_info[option],\
                #                                   this_info[noption])
                if  not 'names' in this_info.dict():
                    this_info.names = fullinfo.parameter.names

                if not option in this_info.dict():
                    this_info[option] = [0]*len(this_info.names)
                if not noption in this_info.dict():
                    this_info[noption] = [0]*len(this_info.names)
                this_info[noption] = assoc_to_flat(this_info.names\
                                        ,this_info[option],\
                                        this_info[noption])   

                ndot = len(fullthis.split('.'))
                pres = ''
                for i in xrange(1,ndot):
                    pres += '  '
                if type(this_info.helper) == str:
                    this_info.helper += "\n%s%-8s = %-8s" % (pres,\
                        fulloption,str(this_info[noption]))
                elif type(this_info.helper) == list:
                    this_info.helper.append("%s%-8s = %-8s" % (pres,\
                        fulloption,str(this_info[noption])))
Exemplo n.º 37
0
class SpecialVariable(ParamStorage):
    '''
    A class that can deal with the datatypes needed for eoldas

    It allows a variable to be set to various data types
    and interprets these into the data structure.
        
    The data structure imposed here is:
        
        self.data.this
        self.name.this 
        
    to store some item called 'this'. Other information can be stored as well
    but part of the idea here is to have some imposed constraints on the
    data structure so that we can sensibly load up odfferent datasets.
        
    The idea is that data will be stored in self.data and associated 
    metadata in self.name. If items are given the same name in both
    sub-structures, we can easily keep track of them. There is no actual
    requirement that this is adhered to, but it is certainy permitted and
    encouraged for the indended use of this class.
        
    Probably the most important thing then about this class is that is
    a SpecialVariable is assigned different data types, then it can do sensible
    things with them in the context of the EOLDAS (and wider applications).
        
    When an assignment takes place (either of ther form 
        
        self.state = foo
        
        or
        
        self['state'] = foo
        
    then what is actually stored depends on the type and nature of foo. The 
    main features as follows:
        
    If foo is a string:
        A guess is made that it is a filename, and an attempt is made to
        read the file. All directories in the list self.dirnames are 
        searched for the filename, and any readable files found are 
        considered candidates, Each of these is read in turn. A set of
        potential data formats, specified by the readers in readers 
        (self.reader_functions) is considered, as if a sucessful interpretation
        takes place the data is returned and stiored in the derired variable.
        
        So, for example, if we have self.state = foo as above and foo is a valid, 
        readable file in the list of directories specified, and it is 
        interprable with one of the formats defined, then the main dataset
        is loaded into:
        
        self.data.state 
        
        (alternatively known as self.data['state']).
        
    If foo is a ParamStorage, it should have the same structure as that here 
        (i.e. self.data and self.name) and these structures are then loaded.
        
        
    If foo is a dictionary (type dict)
        It is first converted to a ParamStorage and then loaded as above.
        
    If foo is any other datatype, it is left pretty much as it, except that
        an attempt to convert to a np.array is made.
        
        Depending on the format, there might be data other than the main
        dataset (e.g. locational information) and these are loaded by the 
        loaders into relevant oarts of self.data and self.name.
        
        For classes that use this class for EOLDAS, we will typically use:
        
        self.data.state     : state variable data
        self.data.sd        : uncertainty information as sd
                              (or a similar fuller representation)
        self.data.control   : control information (e.g. view angles)
        self.data.location  : location information
        
        with associated descriptor data in the relevant parts of self.name.
        
        The idea for simple use of the data structure then is for all of these
        datasets represented as 2D datasets, where the number of rows in
        each of the self.data.state etc field will be the same, but
        the number of columns will tend to vary (e.g different numbers of state
        variables). 
        
        The reason for considering such a 2D 'flat'(ish) representation is
        that it is easy to tabulate and understand. In fact the data will be of
        quite high dimension. E.g. is the data vary by time, x and y, then
        we would have 3 columns for self.data.location, with descriptors for
        the columns in self.name.location, and corresponding state data in
        self.data.state (with the number of state variables determining the
        number of columns in that table).
        
        As mentioned, at the pooint of this class, there is no strict 
        requirement for any such structure ti the data loaded or used, but 
        that is the plan for EOLDAS use, so worth documenting at this point.

    '''

    def __init__(self,info=[],name=None,thisname=None,readers=[],log_terms={},\
                                                datadir=["."],env=None,\
                                                header=None,writers={},
                                                simple=False,logger=None):
        '''
        Class SpecialVariable initialisation.
            
        Sets up the class as a ParamStorage and calls self.init()
            
        See init() fort a fuller descripotion of the options.
            
        '''
        ParamStorage.__init__(self,logger=logger)
        name = name or thisname
        if name == None:
           import time
           thistime = str(time.time())
           name = type(self).__name__
           name =  "%s.%s" % (name,thistime)
        self.thisname = name

        self.init(info=[],name=self.thisname,readers=readers,log_terms={},\
                                                datadir=datadir,env=env,\
                                                header=header,writers=writers,\
                                                simple=False,logger=logger)
        

    
    def init(self,info=[],name=None,readers=[],log_terms={},\
                                                datadir=None,env=None,\
                                                header=None,writers={},\
                                                simple=False,logger=None):
        '''
        Initialise information in a SpecialVariable instance.
            
        Options:
            info        :   Information tthat can be passed through to reader
                            methods (a list).
            thisname    :   a name to use to identify this instance in any
                            logging. By default this is None. If thisname is set
                            to True, then logging is to stdout.
            readers     :   A list of reader methods that are pre-pended to 
                            those already contained in the class.
            log_terms   :   A dictionary of log options. By default
                            {'logfile':None,'logdir':'log','debug':True}
                            If thisname is set, and logfile specified, then 
                            logs are logged to that file. If thisname is set
                            to True, then logging is to stdout.
            datadir     :   A list of directories to search for data files
                            to interpret if the SpecialVariable is set to a 
                            string.
            env         :   An environment variable that can be used to extend
                            the datadir variable.
            header      :   A header string to use to identify pickle files.
                            By default, this is set to
                            "EOLDAS -- plewis -- UCL -- V0.1"
            simple      :   A flag to swicth off the 'complicated' 
                            interpretation methods, i.e. just set and return 
                            variables literally, do not try to interpret them.
            
        
        '''
        self.set('simple',True)
        if name == None:
           import time
           thistime = str(time.time())
           name = type(self).__name__
           name =  "%s.%s" % (name,thistime)
        self.thisname = name

        # this is where we will put any data     
        self.data = ParamStorage()
        self.name = ParamStorage()

        self.info = info

        self.datadir = datadir or ['.']
        self.env = env     

        init_read_write(self,header,readers,writers)
        
        # sort logging and log if thisname != None
        self.log_terms = {'logfile':None,'logdir':'log','debug':True}
        # override logging info
        for (key,value) in log_terms.iteritems():
            self.log_terms[key] = value
        self.logger= sortlog(self,self.log_terms['logfile'],logger,name=self.thisname,\
				logdir=self.log_terms['logdir'],\
				debug=self.log_terms['debug'])
        self.simple = simple

    set = lambda self,this,value :ParamStorage.__setattr__(self,this,value)
    set.__name__ = 'set'
    set.__doc__ = """
        A method to set the literal value of this, rather than attempt
        an interpretation (e.g. used when self.simple is True)
    """
    get = lambda self,this :ParamStorage.__getattr__(self,this)
    get.__name__ = 'get'
    get.__doc__ = """
        A method to get the literal value of this, rather than attempt
        an interpretation (e.g. used when self.simple is True)
        """

    def __setitem__(self,this,value):
        '''
        Variable setting method for style self['this']. 
            
        Interpreted the same as via __setattr__.
        
        '''
        # always set the item
        self.__setattr__(this,value)
    
    def __setattr__(self,this,value):
        '''
        Variable setting method for style self.this
            
        Varies what it does depending on the type of value.
            
        The method interprets and sets the SpecialVariable value:
            
            1.  ParamStorage or SpecialVariable. The data are directly loaded.
                This is one of the most flexible formats for input. It expects
                fields 'data' and/or 'name', which are loaded into self.
                There will normally be a field data.this, where this is 
                the variable name passed here.
            2.  A dictionary, same format as the ParamStorage.
            3.  A tuple, interpreted as (data,name) and loaded accordingly.
            4.  *string* as filename (various formats). An attempt to read the
                string as a file (of a set of formats) is made. If none pass 
                then it it maintained as a string.
            5.  A numpy array (np.array) that is loaded into self.data.this.
            6.  Anything else. Loaded into self.data.this as a numpy array.
            
        '''
        if self.simple:
            self.set(this,value)    
            return
        t = type(value)
        try:
            if t == ParamStorage or t == SpecialVariable:
                # update the whole structure
                #self.__set_if_unset('data',ParamStorage())
                #self.__set_if_unset('name',ParamStorage()) 
                self.data.update(value.data,combine=True)
                self.name.update(value.name,combine=True)
            elif t == dict:
                n_value = ParamStorage().from_dict(value)
                self.__setattr__(this,n_value)
            elif t == tuple or t == list:
                # assumed to be (data,name) or [data,name]
                #self.__set_if_unset('data',ParamStorage())
                #self.__set_if_unset('name',ParamStorage())
                #ParamStorage.__setattr__(self['data'],this,value[0]) 
                #ParamStorage.__setattr__(self['name'],this,value[1])
                ParamStorage.__setattr__(self['data'],this,np.array(value))
            elif t == str:
                # set the term
                #self.__set_if_unset('data',ParamStorage())
                #self.__set_if_unset('name',ParamStorage())
                ParamStorage.__setattr__(self['data'],this,value)
                # interpret as a file read if possible
                self.process_data_string(this,info=self.info)
            elif t == np.ndarray:
                #self.__set_if_unset('data',ParamStorage())
                #self.__set_if_unset('name',ParamStorage())
                ParamStorage.__setattr__(self['data'],this,value)
            else:
                ParamStorage.__setattr__(self['data'],this,\
                                         np.array(value))
        except:
            if self.logger:
                self.logger.info("Failed to set SpecialVariable %s from %s %s"\
                             %(this,t.__name__,value))
            return
        if self.logger:
            self.logger.info("Set variable %s from type %s"%(this,t.__name__))

    def __getattr__(self,name):
        '''
        Variable getting method for style self.this 
            
        If the field 'data' exists in self.__dict__ and 'name'
        is in the dictionary, then the field self.data.this is returned.
        
        Otherwise, if the field 'name' is in self.__dict__, self.name
        is returned.
            
        Otherwise return None.

        '''
        if 'data' in self.__dict__ and name in self.data.__dict__:
            return self.data.__dict__.__getitem__ ( name )
        elif name in self.__dict__:
            return self.__dict__.__getitem__ ( name )
        else:
            return None

    def __getitem__(self,name):
        '''
        Variable getting method for style self['this']. 
            
        Interpreted the same as via __getattr__.
            
        '''
        # first look in data
        if 'data' in self.__dict__ and name in self.data.__dict__:
            return self.data.__dict__.__getitem__ ( name )
        elif name in self.__dict__:
            return self.__dict__.__getitem__ ( name )
        else:
            return None
    

    def process_data_string(self,name,info=[],fmt=None):
        '''
        Attempt to load data from a string, assuming the string is a filename.
            
        The array self.datadir is searched for readable files with the 
        string 'name' (also self.env), and a list of potential files
        considered for reading.
            
        Each readable file is passed to self.read, and if it is interpretable,
        it is loaded according to the read method.
           
        Note tha the format can be specified. If not, then all formats
        are attempted until a sucessful read is made.
 
        '''
        from eoldas_Lib import get_filename

        orig = self.data[name]
        if self.logger:
            self.logger.debug('%s is a string ... see if its a readable file ...' \
                          % name)
        # find a list of potential files
        goodfiles, is_error = get_filename(orig,datadir=self.datadir,\
                                          env=self.env,multiple=True)
        if is_error[0] and self.logger:
            self.logger.debug(str(is_error[1]))
            return 
        if self.logger:
            self.logger.debug("*** looking at potential files %s"%str(goodfiles))
        # loop over all files that it might be
        for goodfile in goodfiles:
            stuff,is_error = reader(self,goodfile,name,fmt=fmt,info=info)
            if not is_error[0] and self.logger:
                self.logger.info("Read file %s "%goodfile)
                return 
        if self.logger:
            self.logger.debug(self.error_msg)
        return 


    write = lambda self,filename,fmt : writer(self,filename,None,fmt=fmt)    
    read  = lambda self,filename,fmt : reader(self,filename,None,fmt=fmt,info=[])  
Exemplo n.º 38
0
    def J_prime_prime(self):
	'''
	Calculation of J''

	We already have the differntial operator
	self.linear.D1 and self.gamma
	after we call self.J_prime()

	Here, J'' = D1.T gamma^2 D1

	J' is of shape (nobs,nstates)
	which is the same as the shape of x
	
	D1 is of shape (nobs,nobs)
	which needs to be expanded to 
	(nobs,nstates,nobs,nstates)

	'''
	x,Cx1,xshape,y,Cy1,yshape = self.getxy()
	J,J_prime = self.J_prime() 
	xshape = self.x.state.shape

	if not 'linear' in self.dict():
	    self.linear = ParamStorage()
	if not 'J_prime_prime' in self.linear.dict():
	    self.linear.J_prime_prime = \
		np.zeros(xshape*2) 
	else:
	    self.linear.J_prime_prime[:] = 0 
	# we need an indexing system in case of multiple
	# nobs columns
        x2a = np.diag(np.ones(self.linear.x2shape[:-1]).flatten())
	try:
	    gamma = self.linear.gamma.flatten()
	except:
            if self.linear.gamma_col != None:
                gamma = x.reshape(self.x.state.shape)\
                    [...,self.linear.gamma_col].flatten()
            else:
                # no gamma variable, so use 1.0 
                gamma = x.reshape(self.x.state.shape)\
                    [...,0].flatten()*0.+1.
            gamma = self.linear.gamma.flatten()
        if self.rt_model.inverse_gamma:
            tgamma = 1./gamma
	    dg = 2./(gamma*gamma*gamma)
        else:
            tgamma = gamma
	    dg = 1.0
        nshape = tuple([np.array(self.linear.x2shape[:-1]).prod()])
        D1 = np.matrix(self.linear.D1.reshape(nshape*2))
	i = 0
        # so, e.g. we have xshape as (50, 100, 2)
        # because one of those columns refers to the gamma value
        # self.linear.gamma_col will typically be 0
	for count in xrange(xshape[-1]):
            if count != self.linear.gamma_col:
                # we only want to process the non gamma col
                C1 = np.diag(self.linear.C1[...,i].\
                             reshape(self.linear.D1.shape[0]))
	        xg = np.matrix(x2a*tgamma*tgamma)
	        dxg = D1 * xg
	        deriv = np.array(dxg.T * C1 * D1)
                # so we have gamma^2 D^2 which is the Hessian
                # we just have to put it in the right place now
                # the technical issue is indexing an array of eg 
                # (50, 100, 2, 50, 100, 2)
                # but it might have more or fewer dimensions
                nd = len(np.array(xshape)[:-1])
                nshape = tuple(np.array(xshape)[:-1])
                if nd == 1:
                    self.linear.J_prime_prime[:,count,:,count] = deriv.reshape(nshape*2)
                elif nd == 2:
                    self.linear.J_prime_prime[:,:,count,:,:,count] = deriv.reshape(nshape*2)
                elif nd == 3:
                    self.linear.J_prime_prime[:,:,:,count,:,:,:,count] = deriv.reshape(nshape*2)
                else:
                    self.logger.error("Can't calculate Hessian for %d dimensions ... I can only do up to 3"%nd)
 
	        #ww = np.where(deriv)
	        #ww2 = tuple([ww[0]]) + tuple([ww[0]*0+count]) \
		#		+ tuple([ww[1]] )+ tuple([ww[0]*0+count])
                #x1 = deriv.shape[0]
                #x2 = self.linear.J_prime_prime.shape[-1]
                #xx = self.linear.J_prime_prime.copy()
                #xx = xx.reshape(x1,x2,x1,x2)
                #xx[ww2] = deriv[ww]
	        #self.linear.J_prime_prime = xx.reshape(self.linear.J_prime_prime.shape)
		i += 1
        if self.linear.gamma_col != None:
            c = self.linear.gamma_col
            nd = len(np.array(xshape)[:-1])
            nshape = tuple(np.array(xshape)[:-1])
            deriv = np.diag(dg*2*J/(tgamma*tgamma)).reshape(nshape*2)
            if nd == 1:
                self.linear.J_prime_prime[:,c,:,c] = deriv
            elif nd == 2:
                self.linear.J_prime_prime[:,:,c,:,:,c] = deriv
            elif nd == 3:
                self.linear.J_prime_prime[:,:,:,c,:,:,:,c] = deriv
            else:
                self.logger.error("Can't calculate Hessian for %d dimensions ... I can only do up to 3"%nd)

	    #dd = np.arange(nshape[0])
            #x1 = dd.shape[0]
            #x2 = self.linear.J_prime_prime.shape[-1]
            #xx = self.linear.J_prime_prime.copy()
            #xx = xx.reshape(x1,x2,x1,x2)
            #xx[dd,dd*0+self.linear.gamma_col,\
            #	dd,dd*0+self.linear.gamma_col] = dg*2*J/(tgamma*tgamma)
            #self.linear.J_prime_prime = xx.reshape(self.linear.J_prime_prime.shape)
	n = np.array(xshape).prod()
	return J,J_prime,self.linear.J_prime_prime.reshape(n,n)
Exemplo n.º 39
0
    def __init__(self,args,name=None,general=None,log=False,logger=None,outdir=".",getopdir=False,parse=True):
        """
        Initialise parser class.

        This sets up the class defaults.
        
        Options:
            
        general=general: this over-rides and defaults with values 
                        set in parser general can be of the form:
                         1. class ParamStorage (i.e. the same form 
                            as self.general)
                         2. a command line list (where the first 
                            item in the list is ignored
                         3. a string containing a set of command line general
            
                         See self.parse() for more details on general 
                            2 and 3 as these simply make a call to tha
                            method.
            
        log=True         If log is set to True, then logging starts 
                            when this class is instanced.
                            Note that the logfile and logdir might 
                            change if subsequent calls to Parser.parse()
                            are made
        """
        if type(args) == str:
            args = args.split()
        self.dolog = log 
        self.log = log
        self.name = args[0]
        self.args = args[1:]
        self.fullargs = args
        self.store_fullargs = args
        if name == None:
           import time
           thistime = str(time.time())
           name = type(self).__name__
           name =  "%s.%s" % (name,thistime)
	self.thisname = name
        # find the following flags:
        # --conf | -c : conf
        # --datadir   : datadir
        datadir = [".","~/.eoldas",sys.path[0]+'/../bin',sys.path[0]+'/../confs',\
		sys.path[0]+'/../system_confs',sys.path[0]+'/../eoldaslib']
        conf = "default.conf"
        logfile = None
        logdir = "."
        self.top = ParamStorage ()
        self.top.general = ParamStorage ()
        self.top.general.__helper__ = ParamStorage ()
        self.top.general.__default__ = ParamStorage ()
        self.top.general.__extras__ = ParamStorage ()
        self.top.general.conf = []
        for i in xrange(len(self.args)):
            theseargs = self.args[i].split('=')
            if theseargs[0] == "--conf":
                conf = theseargs[1]
                self.top.general.conf.append(conf)
            elif theseargs[0][0:2] == "-c":
                if len(theseargs) > 2:
                    conf = theseargs[0][2:]
                else:
                    conf = self.args[i+1]
		self.top.general.conf.append(conf)
            elif theseargs[0] == "--datadir":
                datadir1 = theseargs[1].replace('[','').\
                                            replace(']','').split()
                [datadir1.append(datadir[i]) for i in \
                                            xrange(len(datadir))]
                datadir = datadir1
            elif theseargs[0] == "--logfile":
                logfile= theseargs[1]
            elif theseargs[0] == "--logdir":
                logdir = theseargs[1]
            elif theseargs[0] == "--outdir":
                outdir = theseargs[1]
        if self.top.general.conf == []:
	    self.top.general.conf = conf 
	if logfile == None:
            logfile = conf.replace('conf','log')
        self.top.general.here = os.getcwd()
        self.top.general.datadir = datadir
        self.top.general.logfile = logfile
        self.top.general.logdir = logdir
        self.top.general.outdir = outdir
        # add here to datadir
        # in addition to '.' to take account of the change of directory
        self.top.general.datadir = self.__add_here_to_datadir(\
                    self.top.general.here,self.top.general.datadir)
        self.top.general.datadir = self.__add_here_to_datadir(\
                    self.top.general.outdir,self.top.general.datadir)
        # cd to where the output is to be 
        self.__cd(self.top.general.outdir)
        # set up the default command line options
        self.default_loader() 
        # update with anything passed here
        if general and type(general) == ParamStorage: self.top.update(\
                    self.__unload(general),combine=True)
        # read the conf files to get any cmd line options
        self.logger = sortlog(self,self.top.general.logfile,logger,name=self.thisname,\
                    logdir=self.top.general.logdir)
        self.config = ConfFile(self.top.general.conf,name=self.thisname+'.config',\
		    loaders=self.loaders,datadir=self.top.\
                    general.datadir,logger=self.logger,logdir=self.top.general.logdir,\
                    logfile=self.top.general.logfile)   
        if len(self.config.configs) == 0:
            this = "Warning: Nothing doing ... you haven't set any configuration",\
						self.config.storelog
	    try:
                self.logger(this)
	    except:
	 	print "Called with args:"
		print "eoldas",self.args
	 	pass
            raise Exception(this)
            
        # now loaders contains all of the defaults set here
        # plus those from the config (config opver-rides defaults here)
        self.loaders = self.config.loaders
        # now convert loaders into parser information
        self.parseLoader(self.loaders)
        self.parse(self.fullargs)
        if general and type(general) == ParamStorage: 
		self.top.update(self.__unload(general),combine=True)
        if general and type(general) == str: 
		self.parse(general.split())
        if general and type(general) == list: 
		self.parse(general)
        # now update the info in self.config
        for i in self.config.infos:
            i.update(self.top,combine=True)
            # so now all terms in self.config.infos
            # contain information from the config file, updated by 
            # the cmd line
            i.logger = self.logger
            i.log()
        # move the information up a level
        self.infos = self.config.infos
        self.configs = self.config.configs
        self.config_log = self.config.storelog

        #if getopdir:
        #    self.sortnames()
        self.config.loglist(self.top)

        #del(self.config.infos)
        #del(self.config.configs)
        #del(self.config)
        self.__cd(self.top.general.here)
Exemplo n.º 40
0
class DModel_Operator ( Operator ):
    
    def preload_prepare(self):
        '''
        Here , we use preload_prepare to make sure 
        the x & any y data are gridded for this
        operator. This greatly simplifies the 
        application of the differential operator.
            
        This method is called before any data are loaded, 
        so ensures they are loaded as a grid.
        '''
	from eoldas_Lib import sortopt
        for i in np.array(self.options.datatypes).flatten():
            # mimic setting the apply_grid flag in options
            if  self.dict().has_key('%s_state'%i):
                self['%s_state'%i].options[i].apply_grid = True
        self.novar = sortopt(self,'novar',False)
        self.gamma_col = sortopt(self,'gamma_col',None)
	self.beenHere =False

    def postload_prepare(self):
        '''
        This is called on initialisation, after data have been read in
           
	Here, we load parameters specifically associated with the
	model H(x).

	In the case of this differential operator, there are:

		model_order	: order of the differential operator
				  (integer)
		wraparound	: edge conditions
				  Can be:
					periodic
					none
					reflexive
		lag		: The (time/space) lag at which
				  the finite difference is calculated
				  in the differential operator here.
				  If this is 1, then we take the difference
				  between each sample point and its neighbour.
				  This is what we normally use. The main
				  purpose of this mechanism is to allow
				  differences at multiple lags to be 
			          calculated (fuller autocorrelation function
				  constraints as in kriging)

				  Multiple lags can be specified (which you could
				  use to perform kriging), in which case lag
				  weight should also be specified.

		lag_weight	: The weight associated with each lag. This will
				  generally be decreasing with increasing lag for
				  a 'usual' autocorrelation function. There is no point
				  specifying this if only a single lag is specified
				  as the function is normalised.
				
	If the conditions are specified as periodic
	the period of the function can also be specified, e.g. 
	for time varying data, you could specify 365 for the 
	periodic period.

	These are specified in the configuration file as

		operator.modelt.rt_model.model_order
		operator.modelt.rt_model.wraparound
		operator.modelt.rt_model.lag
		operator.modelt.rt_model.lag_weight

	The default values (set here) are 1, 'none', 1 and 1 respectively.

	To specify the period for `periodic` specify e.g.:

	[operator.modelt.rt_model]
	wraparound=periodic,365

	The default period is set to 0, which implies that it is periodic
	on whatever the data extent is.

	Or for multiple lags:

	[operator.modelt.rt_model]
	lag=1,2,3,4,5
	lag_weight=1,0.7,0.5,0.35,0.2

	NB this lag mechanism has not yet been fully tested
        and should be used with caution. It is intended more
	as a placeholder for future developments.

	Finally, we can also decide to work with 
	inverse gamma (i.e. an uncertainty-based measure)

	This is achieved by setting the flag

		operator.modelt.rt_model.inverse_gamma=True

	This flag should be set if you intend to estimate gamma in the
	Data Assimilation. Again, the is experimental and should be used
	with caution.
 
        '''
        from eoldas_Lib import sortopt
        self.rt_model = sortopt(self.options,'rt_model',ParamStorage())
	self.rt_model.lag = sortopt(self.rt_model,'lag',1)
	self.rt_model.inverse_gamma= \
		sortopt(self.rt_model,'inverse_gamma',False)
        self.rt_model.model_order = \
			sortopt(self.rt_model,'model_order',1)
	self.rt_model.wraparound = \
			sortopt(self.rt_model,'wraparound','none')
        self.rt_model.wraparound_mod = 0
	if np.array(self.rt_model.wraparound).size == 2 and \
		np.array(self.rt_model.wraparound)[0] == 'periodic':
	    self.rt_model.wraparound_mod = \
			np.array(self.rt_model.wraparound)[1]
	    self.rt_model.wraparound = \
                        np.array(self.rt_model.wraparound)[0]
        self.rt_model.lag = \
			sortopt(self.rt_model,'lag',[1])
	self.rt_model.lag = np.array(self.rt_model.lag).flatten()

	self.rt_model.lag_weight = \
			sortopt(self.rt_model,'lag_weight',[1.]*\
			self.rt_model.lag.size)
	self.rt_model.lag_weight = np.array(\
			self.rt_model.lag_weight).flatten().astype(float)

	if self.rt_model.lag_weight.sum() == 0:
	    self.rt_model.lag_weight[:] = np.ones(self.rt_model.lag_weight.size)
 
	self.rt_model.lag_weight = self.rt_model.lag_weight\
			/ self.rt_model.lag_weight.sum()
    
    def setH(self):
        '''
            This method sets up the matrices required for the model.
            
            This operator is written so that it can apply smoothing
            in different dimensions. This is controlled by the model
            state vector. 
            
            The names of the states are stored in self.x_meta.location
            and the associated location information in self.x_meta.location.
            So, we look through these looking for matches, e.g. 'row' in 
            location and 'gamma_row' in names would mean that we want
            to apply the model over the row dimension. There should be only
            one gamma term in the state vectors for this operator. If you
            give more than one, only the last one will be used.
            
            NOT YET IMPLEMENTED: 
            The model can be applied to multiple dimensions by specifying
            e.g. gamma_time_row. If you want separate gammas for e.g. 
            time and row, then you should use separate operators. If
            gamma_roccol is specified, then the model applies to
            Euclidean distance in row/col space.
            
            Formally, the problem can be stated most simply as a matrix
            D so that gamma D x is the rate of change of x with respect 
            to the target location variable (time, row, col etc). 
            The job of this method then is to form and store D.
            
            The main complication to this is we have to split up x into
            those terms that we will apply D to (x2 here) and separately
            pull out the gamma terms. The resultant matrix D then needs to
            be re-formed so as to apply to the whole vector x, rather than 
            just x2. We do this with masks.
            
            On input, x is a 1D vector.

        '''
	x,Cx1,xshape,y,Cy1,yshape = self.getxy()
        # the names of the variables in x
        names = np.array(self.x_meta.state)
        # the names of the location information (e.g. time, row, col)
        location = self.x_meta.location
        self.logger.info('Setting up model matrices...')

        if self.x_meta.is_grid:
            try:
		self.x.location = self.x_state.ungridder.location
		self.x.qlocation = self.x_state.ungridder.qlocation
            except:
                raise Exception("You are trying to ungrid a dataset that wasn't gridded using State.regrid()" +\
                    " so the ungridder information is not available. Either load the data using State.grid " +\
                    " or set it up some other way or avoid calling this method with this type of data")
        
        # first, reshape x from its 1-D form to
        # have the same shape as self.x.state. We store
        # this shape as xshape.
        xshape = self.x.state.shape
        
        # we can't change the tuple directly, so need a 
        # vector representation that we can manipulate
        # This is xshaper
        xshaper = np.array(xshape)
        
        # the data are assumed loaded into x
        
        # At this point, x2 is just a copy of the full input vector x
        # mask then is a mask of the same size as self.x_meta.state
        # by deafult, this mask is True. We will modify it to
        # take out bits we dont want later.
        x2 = x.reshape(xshape)
        mask = np.ones_like(x2).astype(bool)
        
        # We now need to recognise any gamma terms that might be in
        # the state vector. Candidates are 'gamma_%s'%(location)
        # e.g. gamma_time.
        
        # The number of dimensions of x can vary, depending on how many
        # loaction terms are used, so its a little tricky to 
        # pull the information out.
        # We loop over the locations, indexed as i
        self.linear.datamask = np.ones(xshape[-1]).astype(bool)
        for i in xrange(len(location)):
            # and form the name of the candidate term in the variable 'this'
            this = 'gamma_%s'%location[i]
            ww = np.where(this == names)[0]
            # Then we see if it appears in the names of the state variables
            if len(ww):
                # form a mask so we dont apply the operator to gamma
                # terms. Note that *all* gamma terms are masked
                # even though we only actually use the last one we
                # come across. 
                # we use [...,ww[0]] because the identifier for the
                # state is always in the final dimension.
                mask[...,ww[0]] = False
                # We store ww[0] as it will alllow us to access gamma
                # in subsequent calls in this same way. This is
                # self.linear.gamma_col
                self.linear.gamma_col = ww[0]
                # and is used as ...
                gammas = x2[...,self.linear.gamma_col]
                self.linear.datamask[self.linear.gamma_col] = False
                # We want to store an index into which of the 
                # location vector terms we are dealing with here.
                # This is 
                self.linear.gamma_loc = i
                # Once we apply the mask to get rid of the gamma columns
                # we need to keep track of the new shape for x2
                # This will be x2shape
                xshaper[-1] -= 1
        
        self.linear.x2shape = tuple(xshaper)
        self.linear.x2mask = mask.flatten()
        # so, apply the mask to take out the gamma columns
        x2 = x[self.linear.x2mask].reshape(self.linear.x2shape)
    
        # We next need access to the location information
        # for the selected dimension self.linear.gamma_loc.
        # If the data are gridded, we need to form the relevant information
        # Ungridded data we can access location directly as it is explicitly
        # stored. We store the location vector as 'locations'
        try:
            locshape = gammas.shape
        except:
            # If no gamma term is given, it is implicit that it is 
            # the first dimension of location, but we have no data to mask
            self.linear.gamma_col = None
            self.linear.gamma_loc = 0
            locshape = (0)
            gammas = x2[...,0]*0.+1.0
        #if self.x_meta.is_grid:
            # the locational variable of interest is self.linear.gamma_loc
            # the grid is dimensioned e.g. [t,r,c,p]
            # so we need e.g. locations which is of dimension
            # e.g. [t,r,c]
        #    locations = self.x.location
        # access the ungridded location data
        lim = self.x_meta.qlocation[self.linear.gamma_loc]
        nloc = lim[1] - lim[0] + 1
        locations = self.x.location[...,self.linear.gamma_loc]
	locshape = tuple(np.array(self.x.location.shape)[:-1])

	for (i,lag) in enumerate(self.rt_model.lag):
	    wt = self.rt_model.lag_weight[i]
	    
            slocations = wt*(np.roll(locations,lag,\
				axis=self.linear.gamma_loc) - locations).astype(float)
            slocations2 = (locations - np.roll(locations,-lag,\
				axis=self.linear.gamma_loc)).astype(float)
                
            # If there is no variation, it is a waste of time to calculate
            # the derivative
            if i == 0 and np.abs(slocations).sum() + np.abs(slocations2).sum() == 0:
                # there is no variation here    
                self.novar = True
                return 0
            self.novar = False
            ww = np.where(slocations > 0)
	    mod = int(self.rt_model.wraparound_mod)/lim[-1] or slocations.shape[self.linear.gamma_loc]
            if self.rt_model.wraparound == 'reflexive':
                slocations[ww] = 0.
                #slocations[ww] = -np.fmod(mod - slocations[ww],mod)
	    elif self.rt_model.wraparound == 'periodic':
                if self.rt_model.wraparound_mod == 0:
	            slocations[ww] = slocations2[ww]
	        else:
		    slocations[ww] = -np.fmod( mod - slocations[ww],mod)
	    else:   # none
	        slocations[ww] = 0.
            ww = np.where(slocations != 0)
            slocations[ww] = 1./slocations[ww]

	    if i == 0:
        	# Form the D matrix. This is of the size required to 
        	# process the x2 data, and this is the most convenient
       	 	# form to use it in
        	m = np.zeros(slocations.shape * 2)
            ww = np.where(slocations != 0)
	    ww2 = np.array(ww).copy()
	    ww2[self.linear.gamma_loc] = ww2[self.linear.gamma_loc] - lag
	    ww2 = tuple(ww2)
	    m[ww*2] = m[ww*2] - slocations[ww]
   	    if False and self.rt_model.wraparound == 'reflexive':
		ww2 = np.abs(ww-lag)	
	        # this is looped as there might be multiple elements with the	
	        # same index for the reflecxive case
		if m.ndim > 2:
		    raise Exception("Not yet implemented: Can't use reflexive mode for multi-dimensions yet")
                for (c,j) in enumerate(ww2):
                    m[j,ww[c]] = m[j,ww[c]] + slocations[ww[c]]
            else:
		ww = tuple(ww)
		ww2 = tuple(ww2)
	   	m[ww2+ww] = m[ww2+ww] + slocations[ww]
        # fix for edge conditions
        dd = m.copy()
        dd = dd.reshape(tuple([np.array(self.linear.x2shape[:-1]).prod()])*2)
        ddw = np.where(dd.diagonal() == 0)[0]
        for d in (ddw):
            ds = -dd[d,:].sum()
            dd[d,:] += dd[d,:]
            dd[d,d] = ds
        m = dd.reshape(m.shape)
        self.logger.info('Caching model matrices...')
        # 
        if np.array(xshape).prod() == Cy1.size:
            self.linear.C1 = Cy1.reshape(xshape)[mask]\
                    .reshape( self.linear.x2shape )
	elif xshape[1] == Cy1.size:
            self.linear.C1 = np.tile(Cy1,xshape[0])[mask.flatten()].reshape( self.linear.x2shape )
        else:
            raise Exception("Can't deal with full covar matrix in DModel yet")
        nn = slocations.flatten().size
	m = m.reshape(nn,nn) 
        self.linear.D1 = np.matrix(m).T    
	for i in xrange(1,self.rt_model.model_order):
	    m = np.matrix(self.linear.D1).T * m  
        self.linear.D1 = m     
        self.logger.info('... Done')
        return True

    def J(self):
        '''
        A slightly modified J as its efficient to 
        precalculate things for this model
       
	J = 0.5 * x.T D1.T gamma^2 D1 x
 
        '''
        x,Cx1,xshape,y,Cy1,yshape = self.getxy()
        self.Hsetup()
        if self.novar:
            return 0
        xshape = self.x.state.shape
	try:
            if self.linear.gamma_col != None:
                gamma = x.reshape(self.x.state.shape)\
                    [...,self.linear.gamma_col].flatten()
            else:
                # no gamma variable, so use 1.0 
                gamma = x.reshape(self.x.state.shape)\
                    [...,0].flatten()*0.+1.
        except:
	    self.logger.error('gamma_col not set ... recovering and assuming no variation here')
	    self.linear.gamma_col = None
	    gamma = x.reshape(self.x.state.shape)[...,0].flatten()*0.+1.
	    self.novar = True
	    self.Hsetup()
	    return 0
	
        x2 = x[self.linear.x2mask].reshape(self.linear.x2shape)
        J = 0.
        i = 0
	if self.rt_model.inverse_gamma:
	    tgamma = 1./gamma
	else:
	    tgamma = gamma    
        for count in xrange(self.x.state.shape[-1]):
            if count != self.linear.gamma_col:
                C1 = np.diag(self.linear.C1[...,i].\
                             reshape(self.linear.D1.shape[0]))
                x2a = x2[...,i].reshape(self.linear.D1.shape[0])
                xg = np.matrix(x2a*tgamma).T
                dxg = self.linear.D1.T * xg
                
                J += np.array(0.5 * dxg.T * C1 * dxg)[0][0]
                i += 1
    #print x[0],J
        return np.array(J).flatten()[0]
       
    def J_prime_prime(self):
	'''
	Calculation of J''

	We already have the differntial operator
	self.linear.D1 and self.gamma
	after we call self.J_prime()

	Here, J'' = D1.T gamma^2 D1

	J' is of shape (nobs,nstates)
	which is the same as the shape of x
	
	D1 is of shape (nobs,nobs)
	which needs to be expanded to 
	(nobs,nstates,nobs,nstates)

	'''
	x,Cx1,xshape,y,Cy1,yshape = self.getxy()
	J,J_prime = self.J_prime() 
	xshape = self.x.state.shape

	if not 'linear' in self.dict():
	    self.linear = ParamStorage()
	if not 'J_prime_prime' in self.linear.dict():
	    self.linear.J_prime_prime = \
		np.zeros(xshape*2) 
	else:
	    self.linear.J_prime_prime[:] = 0 
	# we need an indexing system in case of multiple
	# nobs columns
        x2a = np.diag(np.ones(self.linear.x2shape[:-1]).flatten())
	try:
	    gamma = self.linear.gamma.flatten()
	except:
            if self.linear.gamma_col != None:
                gamma = x.reshape(self.x.state.shape)\
                    [...,self.linear.gamma_col].flatten()
            else:
                # no gamma variable, so use 1.0 
                gamma = x.reshape(self.x.state.shape)\
                    [...,0].flatten()*0.+1.
            gamma = self.linear.gamma.flatten()
        if self.rt_model.inverse_gamma:
            tgamma = 1./gamma
	    dg = 2./(gamma*gamma*gamma)
        else:
            tgamma = gamma
	    dg = 1.0
        nshape = tuple([np.array(self.linear.x2shape[:-1]).prod()])
        D1 = np.matrix(self.linear.D1.reshape(nshape*2))
	i = 0
        # so, e.g. we have xshape as (50, 100, 2)
        # because one of those columns refers to the gamma value
        # self.linear.gamma_col will typically be 0
	for count in xrange(xshape[-1]):
            if count != self.linear.gamma_col:
                # we only want to process the non gamma col
                C1 = np.diag(self.linear.C1[...,i].\
                             reshape(self.linear.D1.shape[0]))
	        xg = np.matrix(x2a*tgamma*tgamma)
	        dxg = D1 * xg
	        deriv = np.array(dxg.T * C1 * D1)
                # so we have gamma^2 D^2 which is the Hessian
                # we just have to put it in the right place now
                # the technical issue is indexing an array of eg 
                # (50, 100, 2, 50, 100, 2)
                # but it might have more or fewer dimensions
                nd = len(np.array(xshape)[:-1])
                nshape = tuple(np.array(xshape)[:-1])
                if nd == 1:
                    self.linear.J_prime_prime[:,count,:,count] = deriv.reshape(nshape*2)
                elif nd == 2:
                    self.linear.J_prime_prime[:,:,count,:,:,count] = deriv.reshape(nshape*2)
                elif nd == 3:
                    self.linear.J_prime_prime[:,:,:,count,:,:,:,count] = deriv.reshape(nshape*2)
                else:
                    self.logger.error("Can't calculate Hessian for %d dimensions ... I can only do up to 3"%nd)
 
	        #ww = np.where(deriv)
	        #ww2 = tuple([ww[0]]) + tuple([ww[0]*0+count]) \
		#		+ tuple([ww[1]] )+ tuple([ww[0]*0+count])
                #x1 = deriv.shape[0]
                #x2 = self.linear.J_prime_prime.shape[-1]
                #xx = self.linear.J_prime_prime.copy()
                #xx = xx.reshape(x1,x2,x1,x2)
                #xx[ww2] = deriv[ww]
	        #self.linear.J_prime_prime = xx.reshape(self.linear.J_prime_prime.shape)
		i += 1
        if self.linear.gamma_col != None:
            c = self.linear.gamma_col
            nd = len(np.array(xshape)[:-1])
            nshape = tuple(np.array(xshape)[:-1])
            deriv = np.diag(dg*2*J/(tgamma*tgamma)).reshape(nshape*2)
            if nd == 1:
                self.linear.J_prime_prime[:,c,:,c] = deriv
            elif nd == 2:
                self.linear.J_prime_prime[:,:,c,:,:,c] = deriv
            elif nd == 3:
                self.linear.J_prime_prime[:,:,:,c,:,:,:,c] = deriv
            else:
                self.logger.error("Can't calculate Hessian for %d dimensions ... I can only do up to 3"%nd)

	    #dd = np.arange(nshape[0])
            #x1 = dd.shape[0]
            #x2 = self.linear.J_prime_prime.shape[-1]
            #xx = self.linear.J_prime_prime.copy()
            #xx = xx.reshape(x1,x2,x1,x2)
            #xx[dd,dd*0+self.linear.gamma_col,\
            #	dd,dd*0+self.linear.gamma_col] = dg*2*J/(tgamma*tgamma)
            #self.linear.J_prime_prime = xx.reshape(self.linear.J_prime_prime.shape)
	n = np.array(xshape).prod()
	return J,J_prime,self.linear.J_prime_prime.reshape(n,n)

    def J_prime(self):
        '''
            A slightly modified J as its efficient to 
            precalculate things for this model
           
	   J' = D.T gamma^2 D x 
            
        '''
        J = self.J()
        if self.novar:
            return 0,self.nowt
        x,Cx1,xshape,y,Cy1,yshape = self.getxy()
        x2 = x[self.linear.x2mask].reshape(self.linear.x2shape)
        if self.linear.gamma_col != None:
            gamma = x.reshape(self.x.state.shape)\
                [...,self.linear.gamma_col].flatten()
        else:
            # no gamma variable, so use 1.0 
            gamma = x.reshape(self.x.state.shape)\
                [...,0].flatten()*0.+1.
        #gamma = self.linear.gamma.flatten()

        if self.rt_model.inverse_gamma:
            tgamma = 1./gamma
            dg = -1./(gamma*gamma)
        else:
            tgamma = gamma
            dg = 1.0

        g2 = tgamma * tgamma
        xshape = self.x.state.shape
        J_prime = np.zeros((x.shape[0]/xshape[-1],xshape[-1]))
        D2x_sum = 0.
        # loop over the non gamma variables
        i = 0
	# store gamma in case needed elsewhere
	self.linear.gamma = gamma
        for count in xrange(self.x.state.shape[-1]):
            if count != self.linear.gamma_col:
                C1 = np.diag(self.linear.C1[...,i].\
                             reshape(self.linear.D1.shape[0]))
                x2a = x2[...,i].reshape(self.linear.D1.shape[0])
                xg = np.matrix(x2a*tgamma).T
                dxg = self.linear.D1 * xg
                deriv = np.array(dxg.T * C1 * self.linear.D1)[0]
                J_prime[...,count] = deriv * tgamma
                #if self.linear.gamma_col != None:
                #        J_prime_gamma = deriv * x2a
                #        D2x_sum = D2x_sum + J_prime_gamma
                i += 1
        if self.linear.gamma_col != None:    
            J_prime[...,self.linear.gamma_col] = dg*2*J/tgamma
        
        return J,J_prime

    def Hsetup(self):
        '''
        setup for the differential operator H(x) 
            
        '''
        if not self.beenHere and not 'H_prime' in self.linear.dict():
            self.logger.info('Setting up storage for efficient model operator')
            if 'y' in self.dict():
                self.linear.H = np.zeros(self.y.state.shape)
                self.linear.H_prime = np.zeros(self.y.state.shape*2)
            else:
                self.linear.H = np.zeros(self.x.state.shape)
                self.linear.H_prime = np.zeros(self.x.state.shape*2)
            self.setH()
            if self.novar:
                self.nowt = 0. * self.x.state
                #del self.linear.H_prime, self.linear.H
            self.beenHere = True
Exemplo n.º 41
0
    def __min(self,a,b):
        '''
        Min utility for 2 numbers, ignoring None
        '''
        if a == None:
            out = b
        elif b == None:
            out = a
        else:
            out = np.min([a,b])
        if out == None:
            return 0
        else:
            return out     

        # the next critical thing is some observations
        obs = load_brdf_file (brf,self.config,bandpass_names={})
        if obs == False:
            return False

        self.config.operator.obs.update(obs,combine=True)

        # sets up an initial version of x_init
        # which is in the observation 'space' (ie one per obs)
        for n_par in xrange ( self.config.params.n_params ):
            #self.default_vals[n_par] = prior_mean[n_par]
            if np.all( self.obs.x_init[ :, n_par] == 0 ): # No
                self.obs.x_init [ :, n_par ] = self.default_vals [ n_par ]
        # try brfinit_files
        # which can overwrite x_init
        try:
            if self.options.preload != []:
                brfinit_files = self.options.preload
                self.brfinit_files['override'] = brfinit_files
        except:
            if self.options.preload != []:
                self.brfinit_files = ParamStorage ()
                self.brfinit_files['override'] = self.options.preload
                # this is a hack to get the same structure
                self.brfinit_files = self.brfinit_files.dict()
        thisdoys = None
        if self.brfinit_files is not None:
            # this is not consistent with having multiple files
            # and is a bit of a mess 
            for key in self.brfinit_files.keys():
                if type(self.brfinit_files[key]) == type([]):
                    initfile = self.brfinit_files[key][0]
                else:
                    initfile = self.brfinit_files[key]
                #(acovar, abandwidth, abands, anpt, anbands_max, alocation, \
                #    awhichfile, anbands, adoys, aqa, atheta_v, atheta_i,aphi_v, \
                #    aphi_i, aisobs, aobs, aobscovar, aparams_x) = \
                #    load_brdf_file(initfile)
                (thisdoys,thisparams) = self.read_parameters(initfile,confdir=confdir)
                # if fail, thisdoys is None
                #self.obs.x_init[:,:] = aparams_x[:,:]

        if thisdoys == None:
            self.brfinit_files = None
        # For convenience, we can invert the observation covariance matrices
        self.obs.obsinvcovar = []
        self.obs.real_obsinvcovar = []

        for sample_no in xrange( self.obs.npt ):
            temp_mtx = np.matrix( self.obs.obscovar[ sample_no ] ).I
            if self.config.params.scale_cost:
                self.logger.info ("Scaling obs by %f" % \
                    float(self.obs.npt*self.obs.nbands[0] ) )
                self.obs.obsinvcovar.append ( \
                        temp_mtx/float((self.obs.npt*self.obs.nbands[sample_no] )))
            else:
                self.obs.obsinvcovar.append( temp_mtx )
            self.obs.real_obsinvcovar.append (temp_mtx)    

        # if there is anything non zero in x_init, set params_x to that
        if self.obs.x_init.sum() > 0:
            self.params_x = self.obs.x_init.copy()
        else:
            self.params_x = np.zeros ((self.obs.npt, \
                                        self.config.params.n_params))
        # determine which params to fix, based primarily on solve_for flags
        fix_params = define_fixparams(self.parameters, \
            solve_for=self.solve_for,prior_sd=self.prior_sd,model_unc_cfg=self.model_unc_cfg)

        self.config.params.n_model_params = np.sum(fix_params==3) + np.sum(fix_params==4)

        # set up the grid based on the span of unique doys
        self.unique_doys, self.quantised_doys, self.obs_shift = quantise_time ( self.obs.doys, \
                                                self.time_quant ,grid=grid)
        self.grid_n_obs = self.unique_doys.shape[0]

       

        self.fix_params = np.tile(fix_params, self.grid_n_obs).reshape((self.grid_n_obs,self.config.params.n_params))

        self.logger.info ("%d days, %d quantised days" % ( len(self.unique_doys), \
            len(self.quantised_doys) ) )
        self.grid_n_params = fix_params.shape[0]

        # set up a grid model representation from self.params_x
        # we will use then when loading
        # self.params_x is a full representation in obs space
        # so we expand it to the model grid space
        self.store_params = self.get_x(self.params_x,self.fix_params*0.)
 
        # but this may contain zeros if a parameter has not been defined so should be set to the default value
        # or maybe interpolations is better
        udoys = np.unique(self.obs.doys)
        try:
            where_udoys = np.in1d(self.unique_doys,udoys)
        except:
            where_udoys = np.zeros_like(self.unique_doys).astype(np.bool)
            for i in udoys:
                w = np.where(self.unique_doys == i)
                where_udoys[w] = True
        for i in xrange(self.grid_n_params):
            self.store_params[:,i] = np.interp(self.unique_doys,self.unique_doys[where_udoys],self.store_params[where_udoys,i])
        
        # override this with data from brfinit_files
        if self.brfinit_files is not None:
            # zeroth ...
            # pull out elements of thisdoys that appear in  self.unique_doys
            
            # first interpolate thisparams onto the grid
            store_params = self.store_params*0.
            new_thisdoys = np.zeros( self.store_params.shape[0]).astype(np.int)
            # loop over thisdoys and load where appropriate
            for (i,j) in enumerate(thisdoys):
                ww = np.where(j == self.unique_doys)
                store_params[ww,:] = thisparams[i,:] 
                new_thisdoys[ww] = j
            thisdoys = new_thisdoys
            udoys = np.unique(thisdoys)
            try:
                where_udoys = np.in1d(thisdoys,udoys)
            except:
                where_udoys = np.zeros_like(thisdoys).astype(np.bool)
                for i in udoys:
                    w = np.where(where_udoys  == i)
                    where_udoys[w] = True
            for i in xrange(self.grid_n_params):
                self.store_params[:,i] = np.interp(self.unique_doys,self.unique_doys[where_udoys],store_params[where_udoys,i])

        # deal with model uncert
        self.model_unc = np.ones((self.fix_params.shape[1])) 
        for ( i, k ) in enumerate ( self.parameters ):
            if self.model_unc_cfg [ k ] > 0:
                self.model_unc[i] = self.model_unc[i] * self.model_unc_cfg [ k ]
        self.prior_m = np.array([self.prior_mean[k] for k in self.parameters ])
        self.prior_std = np.array([self.prior_sd[k] for k in self.parameters ])

        return #( prior_mean, prior_sd, model_unc, abs_tol, scale_cost)
Exemplo n.º 42
0
    def scan_info(self, config, this, info, fullthis, fullinfo):
        """
        Take a ConfigParser instance config and scan info into 
        config.info. This is called recursively if needed.

        Parameters:
            config    : the configuration object
            this      : the current item to be parsed
            info      : where this item is to go
            fullthis  : the full name of this
            fullinfo  : the full (top level) version of info.
            
        """
        from eoldas_ConfFile import assoc_to_flat
        # find the keys in the top level
        # loop over
        thiss = np.array(this.split('.'))
        # just in case .. is used as separator
        ww = np.where(thiss != '')
        thiss = thiss[ww]
        nextone = ''
        for i in xrange(1, len(thiss) - 1):
            nextone = nextone + thiss[i] + '.'
        if len(thiss) > 1:
            nextone = nextone + thiss[-1]
        # first, check if its already there
        if not hasattr(info, thiss[0]):
            info[thiss[0]] = ParamStorage()
            info[thiss[0]].helper = []
        # load up the info
        if len(thiss) == 1:
            for option in config.options(fullthis):
                fulloption = option
                # option may have a '.' separated term as well
                options = np.array(option.split('.'))
                # tidy up any double dot stuff
                ww = np.where(options != '')
                options = options[ww]
                # need to iterate to make sure it is loaded
                # at the right level
                # of the hierachy
                this_info = info[this]
                # so now this_info is at the base
                for i in xrange(len(options) - 1):
                    if not hasattr(this_info, options[i]):
                        this_info[options[i]] = ParamStorage()
                        this_info[options[i]].helper = []
                    this_info = this_info[options[i]]
                option = options[-1]
                this_info[option] = array_type_convert(fullinfo,\
                                            config.get(fullthis,fulloption))
                if option[:6] == 'assoc_':
                    noption = option[6:]
                    this_info[noption] = assoc_to_flat(\
                                fullinfo.parameter.names,this_info[option],\
                                this_info[noption])
                    is_assoc = True
                else:
                    is_assoc = False
                if not hasattr(this_info, 'helper'):
                    this_info.helper = []
                ndot = len(fullthis.split('.'))
                pres = ''
                for i in xrange(1, ndot):
                    pres += '  '
                if type(this_info.helper) == str:
                    this_info.helper += "\n%s%s.%-8s = %-8s" % \
                        (pres,fullthis,fulloption,str(this_info[option]))
                elif type(this_info.helper) == list:
                    this_info.helper.append("%s%s.%-8s = %-8s" % \
                    (pres,fullthis,fulloption,\
                    str(this_info[option])))
                if is_assoc:
                    if type(this_info.helper) == str:
                        this_info.helper += "\n%s%s.%-8s = %-8s" % \
                                (pres,fullthis,fulloption.replace\
                                ('assoc_',''),str(this_info[noption]))
                    elif type(this_info.helper) == list:
                        this_info.helper.append("%s%s.%-8s = %-8s" % \
                                (pres,fullthis,fulloption.replace\
                                 ('assoc_',''),str(this_info[noption])))
        else:
            self.scan_info(config, nextone, info[thiss[0]], fullthis, fullinfo)
            if thiss[-1][:6] == 'assoc_' and thiss[0] in fullinfo.dict():
                # only do this operation when at the top level
                noption = thiss[-1][6:]
                option = thiss[-1]
                this_info = info
                fulloption = thiss[0]
                this_info = this_info[thiss[0]]
                for i in xrange(1, len(thiss) - 1):
                    this_info = this_info[thiss[i]]
                    fulloption = '%s.%s' % (fulloption, thiss[i])
                fulloption = '%s.%s' % (fulloption, noption)
                #this_info[noption] = assoc_to_flat(fullinfo.parameter.names\
                #                                   ,this_info[option],\
                #                                   this_info[noption])
                if not 'names' in this_info.dict():
                    this_info.names = fullinfo.parameter.names

                if not option in this_info.dict():
                    this_info[option] = [0] * len(this_info.names)
                if not noption in this_info.dict():
                    this_info[noption] = [0] * len(this_info.names)
                this_info[noption] = assoc_to_flat(this_info.names\
                                        ,this_info[option],\
                                        this_info[noption])

                ndot = len(fullthis.split('.'))
                pres = ''
                for i in xrange(1, ndot):
                    pres += '  '
                if type(this_info.helper) == str:
                    this_info.helper += "\n%s%-8s = %-8s" % (pres,\
                        fulloption,str(this_info[noption]))
                elif type(this_info.helper) == list:
                    this_info.helper.append("%s%-8s = %-8s" % (pres,\
                        fulloption,str(this_info[noption])))