Esempio n. 1
0
    def log(self,
            this=None,
            logger=None,
            name=None,
            supername='',
            logfile="log.dat",
            n=0,
            logdir="logs",
            debug=True,
            print_helper=False):
        '''
        Dump the whole contents to the a log

        Useful for inspecting data

        If self.logger is set, this is the log that is used, otherwise
        one is set up from logfile (and possibly logdir).
        In this case, name is used as the identifier in the log. You
        have to set name to enable log initialisation.

        If print_helper is True, then print_helper fields are logged (default False)
        
        '''
        from eoldas_Lib import sortlog
        self.logger = sortlog(self,
                              logfile,
                              logger,
                              name=name,
                              logdir=logdir,
                              debug=debug)
        if this == None:
            this = self
            try:
                self.logger.info("**" * 20)
                self.logger.info("logging parameters ...")
                self.logger.info("**" * 20)
            except:
                pass
        for (k, v) in this.iteritems():
            if supername != '':
                thisname = "%s.%s" % (supername, str(k))
            else:
                thisname = str(k)
            strk = str(k)
            doit = True
            if strk.replace('__', '') != strk or str(k) == 'helper' or str(
                    k)[:5] == "help_":
                doit = False
            if doit or print_helper:
                try:
                    self.logger.info("%s = %s" % (thisname, str(v)))
                except:
                    pass
            if type(v) == ParamStorage and k != 'logger':
                self.log(this=this[k],
                         name=None,
                         n=n + 2,
                         supername=thisname,
                         print_helper=print_helper)
Esempio n. 2
0
    def __init__(self,options,name=None,loaders=[],logfile=None,\
                 logdir=None,datadir=['.','~/.eoldas'],logger=None\
                 ,env="EOLDAS",fatal=False):
        '''
        Parse the configuration file(s) referred to in the (string or list) 
        "options".

        Parameters:
        
            options     :   The class is initialised with "options", 
                            which may be of type ParamStorage
                            or a list or a string.
                            "options" may also be of type ParamStorage, 
                            in which case it is simply loaded here.

        Options:
            
            log_name    :   name of log. If none is specified, 
                            logging goes to stdout.
            log         :   boolean, specifying whether or not to do 
                            logging.
            datadir     :   a list of strings, specifying where to 
                            look for configuration files.
            env         :   an environment variable that may contain 
                            more places to search for configuration files 
                            (after datadir) (default "EOLDAS").
            fatal       :   specify whether not finding a configuration 
                            file should be fatal or not (default False)

        See type_convert() for some rules on defining a configration file.
            
        '''
        from eoldas_Lib import sortlog
        if name == None:
            import time
            thistime = str(time.time())
            name = type(self).__name__
            name = "%s.%s" % (name, thistime)
        self.thisname = name
        self.logdir = logdir
        self.logfile = logfile
        self.storelog = ""
        self.datadir = datadir
        self.env = env
        self.fatal = fatal
        self.logger = sortlog(self,
                              self.logfile,
                              logger,
                              name=self.thisname,
                              logdir=self.logdir,
                              debug=True)
        self.configs = []
        self.infos = []
        self.loaders = loaders
        if type(options) == ParamStorage:
            self.options.update(options, combine=True)
        if type(options) == list or type(options) == str:
            self.read_conf_file(options)
Esempio n. 3
0
    def __init__(self,options,name=None,loaders=[],logfile=None,\
                 logdir=None,datadir=['.','~/.eoldas'],logger=None\
                 ,env="EOLDAS",fatal=False):
        '''
        Parse the configuration file(s) referred to in the (string or list) 
        "options".

        Parameters:
        
            options     :   The class is initialised with "options", 
                            which may be of type ParamStorage
                            or a list or a string.
                            "options" may also be of type ParamStorage, 
                            in which case it is simply loaded here.

        Options:
            
            log_name    :   name of log. If none is specified, 
                            logging goes to stdout.
            log         :   boolean, specifying whether or not to do 
                            logging.
            datadir     :   a list of strings, specifying where to 
                            look for configuration files.
            env         :   an environment variable that may contain 
                            more places to search for configuration files 
                            (after datadir) (default "EOLDAS").
            fatal       :   specify whether not finding a configuration 
                            file should be fatal or not (default False)

        See type_convert() for some rules on defining a configration file.
            
        '''
        from eoldas_Lib import sortlog 
        if name == None:
           import time
           thistime = str(time.time())
           name = type(self).__name__
           name =  "%s.%s" % (name,thistime)
        self.thisname = name
        self.logdir = logdir
        self.logfile = logfile
        self.storelog = ""
        self.datadir=datadir 
        self.env=env
        self.fatal=fatal
        self.logger = sortlog(self,self.logfile,logger,name=self.thisname,logdir=self.logdir,debug=True)
        self.configs = []
        self.infos = []
        self.loaders = loaders
        if type(options) == ParamStorage:
            self.options.update(options,combine=True)
        if type(options)== list or type(options) == str:
            self.read_conf_file(options)
    def log(self,this=None,logger=None,name=None,supername='',logfile="log.dat",n=0,logdir="logs",debug=True,print_helper=False):
        '''
        Dump the whole contents to the a log

        Useful for inspecting data

        If self.logger is set, this is the log that is used, otherwise
        one is set up from logfile (and possibly logdir).
        In this case, name is used as the identifier in the log. You
        have to set name to enable log initialisation.

        If print_helper is True, then print_helper fields are logged (default False)
        
        '''
        from eoldas_Lib import sortlog
        self.logger = sortlog(self,logfile,logger,name=name,logdir=logdir,debug=debug)
        if this == None:
            this = self
            try:
                self.logger.info("**"*20)
                self.logger.info("logging parameters ...")
                self.logger.info("**"*20)
            except:
                pass
        for ( k, v ) in this.iteritems():
            if supername != '':
                thisname = "%s.%s" % (supername,str(k))
            else:
                thisname = str(k)
            strk = str(k)
            doit = True
            if strk.replace('__','') != strk or str(k) == 'helper' or str(k)[:5] == "help_":
                doit = False
            if doit  or print_helper:
                try:
                    self.logger.info("%s = %s" % (thisname,str(v)))
                except:
                    pass
            if type(v) == ParamStorage and k != 'logger':
                self.log(this=this[k],name=None,n=n+2,supername=thisname,print_helper=print_helper)
Esempio n. 5
0
    def __init__(self,args,name=None,general=None,log=False,logger=None,outdir=".",getopdir=False,parse=True):
        """
        Initialise parser class.

        This sets up the class defaults.
        
        Options:
            
        general=general: this over-rides and defaults with values 
                        set in parser general can be of the form:
                         1. class ParamStorage (i.e. the same form 
                            as self.general)
                         2. a command line list (where the first 
                            item in the list is ignored
                         3. a string containing a set of command line general
            
                         See self.parse() for more details on general 
                            2 and 3 as these simply make a call to tha
                            method.
            
        log=True         If log is set to True, then logging starts 
                            when this class is instanced.
                            Note that the logfile and logdir might 
                            change if subsequent calls to Parser.parse()
                            are made
        """
        if type(args) == str:
            args = args.split()
        self.dolog = log 
        self.log = log
        self.name = args[0]
        self.args = args[1:]
        self.fullargs = args
        self.store_fullargs = args
        if name == None:
           import time
           thistime = str(time.time())
           name = type(self).__name__
           name =  "%s.%s" % (name,thistime)
	self.thisname = name
        # find the following flags:
        # --conf | -c : conf
        # --datadir   : datadir
        datadir = [".","~/.eoldas",sys.path[0]+'/../bin',sys.path[0]+'/../confs',\
		sys.path[0]+'/../system_confs',sys.path[0]+'/../eoldaslib']
        conf = "default.conf"
        logfile = None
        logdir = "."
        self.top = ParamStorage ()
        self.top.general = ParamStorage ()
        self.top.general.__helper__ = ParamStorage ()
        self.top.general.__default__ = ParamStorage ()
        self.top.general.__extras__ = ParamStorage ()
        self.top.general.conf = []
        for i in xrange(len(self.args)):
            theseargs = self.args[i].split('=')
            if theseargs[0] == "--conf":
                conf = theseargs[1]
                self.top.general.conf.append(conf)
            elif theseargs[0][0:2] == "-c":
                if len(theseargs) > 2:
                    conf = theseargs[0][2:]
                else:
                    conf = self.args[i+1]
		self.top.general.conf.append(conf)
            elif theseargs[0] == "--datadir":
                datadir1 = theseargs[1].replace('[','').\
                                            replace(']','').split()
                [datadir1.append(datadir[i]) for i in \
                                            xrange(len(datadir))]
                datadir = datadir1
            elif theseargs[0] == "--logfile":
                logfile= theseargs[1]
            elif theseargs[0] == "--logdir":
                logdir = theseargs[1]
            elif theseargs[0] == "--outdir":
                outdir = theseargs[1]
        if self.top.general.conf == []:
	    self.top.general.conf = conf 
	if logfile == None:
            logfile = conf.replace('conf','log')
        self.top.general.here = os.getcwd()
        self.top.general.datadir = datadir
        self.top.general.logfile = logfile
        self.top.general.logdir = logdir
        self.top.general.outdir = outdir
        # add here to datadir
        # in addition to '.' to take account of the change of directory
        self.top.general.datadir = self.__add_here_to_datadir(\
                    self.top.general.here,self.top.general.datadir)
        self.top.general.datadir = self.__add_here_to_datadir(\
                    self.top.general.outdir,self.top.general.datadir)
        # cd to where the output is to be 
        self.__cd(self.top.general.outdir)
        # set up the default command line options
        self.default_loader() 
        # update with anything passed here
        if general and type(general) == ParamStorage: self.top.update(\
                    self.__unload(general),combine=True)
        # read the conf files to get any cmd line options
        self.logger = sortlog(self,self.top.general.logfile,logger,name=self.thisname,\
                    logdir=self.top.general.logdir)
        self.config = ConfFile(self.top.general.conf,name=self.thisname+'.config',\
		    loaders=self.loaders,datadir=self.top.\
                    general.datadir,logger=self.logger,logdir=self.top.general.logdir,\
                    logfile=self.top.general.logfile)   
        if len(self.config.configs) == 0:
            this = "Warning: Nothing doing ... you haven't set any configuration",\
						self.config.storelog
	    try:
                self.logger(this)
	    except:
	 	print "Called with args:"
		print "eoldas",self.args
	 	pass
            raise Exception(this)
            
        # now loaders contains all of the defaults set here
        # plus those from the config (config opver-rides defaults here)
        self.loaders = self.config.loaders
        # now convert loaders into parser information
        self.parseLoader(self.loaders)
        self.parse(self.fullargs)
        if general and type(general) == ParamStorage: 
		self.top.update(self.__unload(general),combine=True)
        if general and type(general) == str: 
		self.parse(general.split())
        if general and type(general) == list: 
		self.parse(general)
        # now update the info in self.config
        for i in self.config.infos:
            i.update(self.top,combine=True)
            # so now all terms in self.config.infos
            # contain information from the config file, updated by 
            # the cmd line
            i.logger = self.logger
            i.log()
        # move the information up a level
        self.infos = self.config.infos
        self.configs = self.config.configs
        self.config_log = self.config.storelog

        #if getopdir:
        #    self.sortnames()
        self.config.loglist(self.top)

        #del(self.config.infos)
        #del(self.config.configs)
        #del(self.config)
        self.__cd(self.top.general.here)
Esempio n. 6
0
    def init(self,info=[],name=None,readers=[],log_terms={},\
                                                datadir=None,env=None,\
                                                header=None,writers={},\
                                                simple=False,logger=None):
        '''
        Initialise information in a SpecialVariable instance.
            
        Options:
            info        :   Information tthat can be passed through to reader
                            methods (a list).
            thisname    :   a name to use to identify this instance in any
                            logging. By default this is None. If thisname is set
                            to True, then logging is to stdout.
            readers     :   A list of reader methods that are pre-pended to 
                            those already contained in the class.
            log_terms   :   A dictionary of log options. By default
                            {'logfile':None,'logdir':'log','debug':True}
                            If thisname is set, and logfile specified, then 
                            logs are logged to that file. If thisname is set
                            to True, then logging is to stdout.
            datadir     :   A list of directories to search for data files
                            to interpret if the SpecialVariable is set to a 
                            string.
            env         :   An environment variable that can be used to extend
                            the datadir variable.
            header      :   A header string to use to identify pickle files.
                            By default, this is set to
                            "EOLDAS -- plewis -- UCL -- V0.1"
            simple      :   A flag to swicth off the 'complicated' 
                            interpretation methods, i.e. just set and return 
                            variables literally, do not try to interpret them.
            
        
        '''
        self.set('simple',True)
        if name == None:
           import time
           thistime = str(time.time())
           name = type(self).__name__
           name =  "%s.%s" % (name,thistime)
        self.thisname = name

        # this is where we will put any data     
        self.data = ParamStorage()
        self.name = ParamStorage()

        self.info = info

        self.datadir = datadir or ['.']
        self.env = env     

        init_read_write(self,header,readers,writers)
        
        # sort logging and log if thisname != None
        self.log_terms = {'logfile':None,'logdir':'log','debug':True}
        # override logging info
        for (key,value) in log_terms.iteritems():
            self.log_terms[key] = value
        self.logger= sortlog(self,self.log_terms['logfile'],logger,name=self.thisname,\
				logdir=self.log_terms['logdir'],\
				debug=self.log_terms['debug'])
        self.simple = simple
Esempio n. 7
0
    def reinit(self,options,names=None,datatype=None,limits=None,\
            bounds=None,control=None,location=None,env=None,header=None,\
               logdir=None,writers={},grid=False,logger=None,\
            datadir=None,logfile=None,name=None,info=[],readers=[],debug=None):
        '''
        Method to re-initialise the class instance
            
        The setup is on the whole controlled by the datatype which
            contains e.g. 'x'. This is used to set up the members
            self.x and self.y as SpecialVariables (see SpecialVariable
            in eoldas_SpecialVariable.py). There are some special attributes
            for datatypes starting with 'y'. These are assumed to be 
            observational data, which means that when they are read, the 
            data names associated with them are not limited to those in 
            self.names but rather set to whatever is read in in the data. This 
            is because the data names for observational data may be terms such 
            as waveband names etc that need special interpretation. Also,
            the default output format for observational data is 
            different to that of other data.  
            
        The elements self.state is a SpecialVariables which means 
            that they can be assigned various data types (see SpecialVariables)
            and loaded accordingly (e.g. if a filename is specified, this is 
            read in to the data structure. The SpecialVariables contain 'hidden'
            datasets, which here are mainly the 'control' and 'location'
            information. A SpecialVariable has two internal structures: `data`
            and `name`. The former is used to store data values (e.g. the 
            state values) and the latter to store associated metadata.
            For example, `control` is passed here e.g. as [`mask`,`vza`] and this
            gives the metadata that are stored in `name`. The actual values
            of the control data are stored in the `data` section. For
            location, we might be passed [`time`,`row`,`col`], so this is set
            in names.location, and the data.location contains the values
            of the location at each of these elements. For the actual state
            dataset, this is stored according to its name, so for `x` the values
            are stored in data.x and the associated data names in name.x.
        
        State datasets must represent at least the mean and standard deviation
            of a state for them to be of value in EOLDAS. TThe mean is accessed
            as e.g. self.state for the state dataset. 
            The sd is accessed can be accessed as self._state.sd if it 
            has been set. This reference can also be used to directly set 
            data associated with a SpecialVariable, e.g.
            self.Data.control = np.zeros([2,3])
            
            to represent 2 samples with 3 control variables. You can access name
            information similarly with
            
            print self.Name.control
            
            but this will generate a KeyError if the term has not been set. You
            can check it exists with:
            
            key = 'control'
            if key in self.Name:
                this = (self.Data[key],self.Name[key])
            
            To get e.g. a dictionary representation of a SpecialVariable
            you can use eg:
            
                self.Name.to_dict()
            
            to get the name dictionary, or
            
                thisdict = self._state.to_dict()
            
            to get the full representation, which then contains 'data' and
            'name' as well as some other information stored in the 
            SpecialVariable.
            
            You can similarly load them using e.g. 
            
                self.Data.update(
                        ParamStorage().from_dict(thisdict['data'])
                        combine=True)
        
        '''

        # set up a fakes dictionary from the data types
        self.set('datatype',datatype)
        self.set('fakes', {'state':'_state'})
        
        # first check that options is sensible
        self.__check_type(options,ParamStorage,fatal=True)
        self.options = options
       
        from eoldas_Lib import set_default_limits,\
                    check_limits_valid,quantize_location, sortopt
   
        nSpecial = 1
        if name == None:
           import time
           thistime = str(time.time())
           name = type(self).__name__
           name =  "%s.%s" % (name,thistime)
        self.thisname = name
        self.options.thisname = str(name).replace(' ','_')
        

        log_terms = {\
            'logfile':logfile or sortopt(self.options,'logfile',None),\
            'logdir':logdir or sortopt(self.options,'logdir',None),\
            'debug' : debug or sortopt(self.options,'debug',True)}
        
        self.datadir = datadir or sortopt(self.options,'datadir',["."])
        self.header = header or "EOLDAS pickle V1.0 - plewis"
        env = env or sortopt(self.options,'env',None)
        names = names or sortopt(self.options,'names',None)
        location = location or sortopt(self.options,'location',['time'])
        control = control or sortopt(self.options,'control',[])
        limits = limits or sortopt(self.options,'limits',\
                        set_default_limits(np.array(location)))
        limits = limits or self.options.limits
        limits = np.array(check_limits_valid(limits))
        bounds = bounds or sortopt(self.options,'bounds',\
                    [[None,None]] * xlen(names))
        self.options.bounds = bounds
       
        self.headers = {'PARAMETERS-V2':"PARAMETERS-V2", \
            'PARAMETERS':"PARAMETERS", \
            'BRDF-UCL':'BRDF-UCL',\
            'BRDF': 'BRDF'}
        self.headers_2 = {'BRDF-UCL':'location'}

        #  The ones pre-loaded are
        # self.read_functions = [self.read_pickle,self.read_numpy_fromfile]
        self._state = SpecialVariable(info=info,name=self.thisname,\
                                  readers=readers,datadir=self.datadir,\
                                  env=env,writers=writers,\
                                  header=self.header,\
                                  logger=logger,log_terms=log_terms,\
                                  simple=False)
        # self._state is where data are read into
        # but self.Data and self.Name are where we access them from
        self.grid=grid
        # this is so we can access this object from
        # inside a SpecialVariable
        self.state = np.array([0.])
        # a default data fmt output     
        if datatype[0] == 'y':
            self.Name.fmt = 'BRDF'
            self.Name.state = np.array(['dummy'])
        else:
            self.Name.fmt = 'PARAMETERS'
            n_params = xlen(names)
            if not n_params:
                error_msg = \
                    "The field 'names' must be defined in options or"+ \
                    "passed directly to this method if you have the data type x"
                raise Exception(error_msg)
        self.Name.state = np.array(names)   
        self.Name.location = np.array(location)
        self.Name.control = np.array(control)
        self.Name.header = self.header
        self.Name.bounds = np.array(bounds)
        self.Name.qlocation = np.array(limits)
        self.Name.datadir = datadir
        #
        # sort this object's name 
        # sort logging
        self.logger = sortlog(self,log_terms['logfile'],logger,name=self.thisname,
			logdir=log_terms['logdir'],debug=log_terms['debug'])
        self.logger.info('Initialising %s' % type(self).__name__)
Esempio n. 8
0
    def __init__(self,
                 args,
                 name=None,
                 general=None,
                 log=False,
                 logger=None,
                 outdir=".",
                 getopdir=False,
                 parse=True):
        """
        Initialise parser class.

        This sets up the class defaults.
        
        Options:
            
        general=general: this over-rides and defaults with values 
                        set in parser general can be of the form:
                         1. class ParamStorage (i.e. the same form 
                            as self.general)
                         2. a command line list (where the first 
                            item in the list is ignored
                         3. a string containing a set of command line general
            
                         See self.parse() for more details on general 
                            2 and 3 as these simply make a call to tha
                            method.
            
        log=True         If log is set to True, then logging starts 
                            when this class is instanced.
                            Note that the logfile and logdir might 
                            change if subsequent calls to Parser.parse()
                            are made
        """
        if type(args) == str:
            args = args.split()
        self.dolog = log
        self.log = log
        self.name = args[0]
        self.args = args[1:]
        self.fullargs = args
        self.store_fullargs = args
        if name == None:
            import time
            thistime = str(time.time())
            name = type(self).__name__
            name = "%s.%s" % (name, thistime)
        self.thisname = name
        # find the following flags:
        # --conf | -c : conf
        # --datadir   : datadir
        datadir = [".","~/.eoldas",sys.path[0]+'/../bin',sys.path[0]+'/../confs',\
  sys.path[0]+'/../system_confs',sys.path[0]+'/../eoldaslib']
        conf = "default.conf"
        logfile = None
        logdir = "."
        self.top = ParamStorage()
        self.top.general = ParamStorage()
        self.top.general.__helper__ = ParamStorage()
        self.top.general.__default__ = ParamStorage()
        self.top.general.__extras__ = ParamStorage()
        self.top.general.conf = []
        for i in xrange(len(self.args)):
            theseargs = self.args[i].split('=')
            if theseargs[0] == "--conf":
                conf = theseargs[1]
                self.top.general.conf.append(conf)
            elif theseargs[0][0:2] == "-c":
                if len(theseargs) > 2:
                    conf = theseargs[0][2:]
                else:
                    conf = self.args[i + 1]
                self.top.general.conf.append(conf)
            elif theseargs[0] == "--datadir":
                datadir1 = theseargs[1].replace('[','').\
                                            replace(']','').split()
                [datadir1.append(datadir[i]) for i in \
                                            xrange(len(datadir))]
                datadir = datadir1
            elif theseargs[0] == "--logfile":
                logfile = theseargs[1]
            elif theseargs[0] == "--logdir":
                logdir = theseargs[1]
            elif theseargs[0] == "--outdir":
                outdir = theseargs[1]
        if self.top.general.conf == []:
            self.top.general.conf = conf
        if logfile == None:
            logfile = conf.replace('conf', 'log')
        self.top.general.here = os.getcwd()
        self.top.general.datadir = datadir
        self.top.general.logfile = logfile
        self.top.general.logdir = logdir
        self.top.general.outdir = outdir
        # add here to datadir
        # in addition to '.' to take account of the change of directory
        self.top.general.datadir = self.__add_here_to_datadir(\
                    self.top.general.here,self.top.general.datadir)
        self.top.general.datadir = self.__add_here_to_datadir(\
                    self.top.general.outdir,self.top.general.datadir)
        # cd to where the output is to be
        self.__cd(self.top.general.outdir)
        # set up the default command line options
        self.default_loader()
        # update with anything passed here
        if general and type(general) == ParamStorage:            self.top.update(\
self.__unload(general),combine=True)
        # read the conf files to get any cmd line options
        self.logger = sortlog(self,self.top.general.logfile,logger,name=self.thisname,\
                    logdir=self.top.general.logdir)
        self.config = ConfFile(self.top.general.conf,name=self.thisname+'.config',\
      loaders=self.loaders,datadir=self.top.\
                    general.datadir,logger=self.logger,logdir=self.top.general.logdir,\
                    logfile=self.top.general.logfile)
        if len(self.config.configs) == 0:
            this = "Warning: Nothing doing ... you haven't set any configuration",\
      self.config.storelog
            try:
                self.logger(this)
            except:
                print "Called with args:"
                print "eoldas", self.args
                pass
            raise Exception(this)

        # now loaders contains all of the defaults set here
        # plus those from the config (config opver-rides defaults here)
        self.loaders = self.config.loaders
        # now convert loaders into parser information
        self.parseLoader(self.loaders)
        self.parse(self.fullargs)
        if general and type(general) == ParamStorage:
            self.top.update(self.__unload(general), combine=True)
        if general and type(general) == str:
            self.parse(general.split())
        if general and type(general) == list:
            self.parse(general)
        # now update the info in self.config
        for i in self.config.infos:
            i.update(self.top, combine=True)
            # so now all terms in self.config.infos
            # contain information from the config file, updated by
            # the cmd line
            i.logger = self.logger
            i.log()
        # move the information up a level
        self.infos = self.config.infos
        self.configs = self.config.configs
        self.config_log = self.config.storelog

        #if getopdir:
        #    self.sortnames()
        self.config.loglist(self.top)

        #del(self.config.infos)
        #del(self.config.configs)
        #del(self.config)
        self.__cd(self.top.general.here)
Esempio n. 9
0
    def __init__(self,confs,logger=None,logfile=None,thisname=None,name=None,datadir=None,logdir=None):
        '''
        Initialise the solver.
            
        This does the following:
            1. Read configuration file(s)
            2. Load operators
            3. Test the call to the cost function
            
        There can be multiple groups of configuration files, so
        self.confs, that holds the core information setup here
        can contain multiple configurations.
        
        The number of configurations is len(confs.infos) and
        the ith configuration is conf = self.confs.infos[i].
            
        Various loggers are available throughout the classes
        used, but the top level logger is self.confs.logger,
        so you can log with e.g.
            
        self.confs.logger.info('this is some info')
            
        The root operator is stored in self.confs.root[i]
        for the ith configuration, so the basic call to the cost
        function is:
            
        J,J_prime = self.confs[i].parameter.cost(None)
            
        '''
        from eoldas_ConfFile import ConfFile
        from eoldas_Lib import sortopt
        name = name or thisname
        if name == None:
           import time
           thistime = str(time.time())
           name = type(self).__name__
           name =  "%s.%s" % (name,thistime)
        self.thisname = name
                
        self.confs = confs
        self.top = sortopt(self,'top',ParamStorage())
        self.top.general = sortopt(self.top,'general',ParamStorage())
        thisname = sortopt(self.top.general,'name',thisname or self.thisname)
        logfile = sortopt(self.top.general,'logfile',logfile or 'log.dat')
        logdir = sortopt(self.top.general,'logdir',logfile or 'logs')
        datadir = sortopt(self.top.general,'datadir',datadir or ['.'])
        self.logger = sortlog(self,logfile,logger or self.confs.logger,\
			name=self.thisname,logdir=logdir,debug=True)
        n_configs = len(self.confs.infos)
        self.confs.root = []
        self.have_unc = False
        try:
            logdir = logdir
        except:
            logdir = self.top.general.logdir

        # first set up parameter
        conf = confs.infos[0]
           
        general = conf.general
        op = conf.parameter    
        if not 'parameter' in conf.dict():
            raise Exception('No parameter field found in %s item %d'%\
                                (conf.__doc__,0))
        general.is_spectral = sortopt(general,'is_spectral',True)
        if not general.is_spectral:
            sort_non_spectral_model(op,conf.operator,logger=confs.logger)
        general.init_test = sortopt(general,'init_test',False)    
        confs.logger.info('loading parameter state')
        conf.parameter.name = 'Operator'
        parameter = eval(op.name)(op,general,\
                                      parameter=None,\
                                      logger=confs.logger,\
                                      name=name+".parameter",\
                                      datatype=list(conf.parameter.datatypes),\
                                      logdir=logdir,\
                                      logfile=logfile,\
                                      datadir=datadir)
        try:
            parameter.transform = parameter.options.x.transform
            parameter.invtransform = parameter.options.x.invtransform
        except:
            parameter.transform = parameter.options.x.names
            parameter.invtransform = parameter.options.x.names
            
        # we now have access to parameter.x.state, parameter.x.sd etc
        # and possibly parameter.y.state etc.
        operators = []
        for (opname,op) in conf.operator.dict().iteritems():
            if opname != 'helper':
	        #pdb.set_trace()
                exec('from eoldas_%s import %s'%(op.name,op.name))
                # make sure the data limits and x bounds are the same 
                # ... inherit from parameter
                op.limits = parameter.options.limits
                if not 'datatypes' in op.dict():
                    op.datatypes = 'x'
                thisop = eval(op.name)(op,general,parameter=parameter,\
                                           logger=confs.logger,\
                                           name=name+".%s-%s"%(thisname,opname),\
                                           datatype=list(op.datatypes),\
                                           logdir=logdir,\
                                           logfile=logfile,\
                                           datadir=datadir)
                # load from parameter
                thisop.loader(parameter)
                operators.append(thisop)
                try:
                    thisop.transform = parameter.options.x.transform
                    thisop.invtransform = parameter.options.x.invtransform
                except:
                    thisop.transform = parameter.options.x.names
                    thisop.invtransform = parameter.options.x.names
		thisop.ploaderMask = np.in1d(parameter.x_meta.state,thisop.x_meta.state)
		try:
		    thisop.invtransform = np.array(thisop.invtransform[thisop.ploaderMask])
		    thisop.transform = np.array(thisop.transform[thisop.ploaderMask])
		except:
		    ww = thisop.ploaderMask
		    thisop.invtransform = np.array(thisop.transform)[ww]
		    thisop.transform = np.array(thisop.transform)[ww]
        # sort the loaders
        parameter.operators = operators
        self.confs.root.append(parameter)
        # Now we have set up the operators
        # try out the cost function
	if general.init_test:    
            self.logger.info('testing cost function calls')
            J,J_prime = self.confs.root[0].cost()
            self.logger.info('done')
        self.confs.root = np.array(self.confs.root)
Esempio n. 10
0
    def __init__(self,
                 confs,
                 logger=None,
                 logfile=None,
                 thisname=None,
                 name=None,
                 datadir=None,
                 logdir=None):
        '''
        Initialise the solver.
            
        This does the following:
            1. Read configuration file(s)
            2. Load operators
            3. Test the call to the cost function
            
        There can be multiple groups of configuration files, so
        self.confs, that holds the core information setup here
        can contain multiple configurations.
        
        The number of configurations is len(confs.infos) and
        the ith configuration is conf = self.confs.infos[i].
            
        Various loggers are available throughout the classes
        used, but the top level logger is self.confs.logger,
        so you can log with e.g.
            
        self.confs.logger.info('this is some info')
            
        The root operator is stored in self.confs.root[i]
        for the ith configuration, so the basic call to the cost
        function is:
            
        J,J_prime = self.confs[i].parameter.cost(None)
            
        '''
        from eoldas_ConfFile import ConfFile
        from eoldas_Lib import sortopt
        name = name or thisname
        if name == None:
            import time
            thistime = str(time.time())
            name = type(self).__name__
            name = "%s.%s" % (name, thistime)
        self.thisname = name

        self.confs = confs
        self.top = sortopt(self, 'top', ParamStorage())
        self.top.general = sortopt(self.top, 'general', ParamStorage())
        thisname = sortopt(self.top.general, 'name', thisname or self.thisname)
        logfile = sortopt(self.top.general, 'logfile', logfile or 'log.dat')
        logdir = sortopt(self.top.general, 'logdir', logfile or 'logs')
        datadir = sortopt(self.top.general, 'datadir', datadir or ['.'])
        self.logger = sortlog(self,logfile,logger or self.confs.logger,\
   name=self.thisname,logdir=logdir,debug=True)
        n_configs = len(self.confs.infos)
        self.confs.root = []
        self.have_unc = False
        try:
            logdir = logdir
        except:
            logdir = self.top.general.logdir

        # first set up parameter
        conf = confs.infos[0]

        general = conf.general
        op = conf.parameter
        if not 'parameter' in conf.dict():
            raise Exception('No parameter field found in %s item %d'%\
                                (conf.__doc__,0))
        general.is_spectral = sortopt(general, 'is_spectral', True)
        if not general.is_spectral:
            sort_non_spectral_model(op, conf.operator, logger=confs.logger)
        general.init_test = sortopt(general, 'init_test', False)
        confs.logger.info('loading parameter state')
        conf.parameter.name = 'Operator'
        parameter = eval(op.name)(op,general,\
                                      parameter=None,\
                                      logger=confs.logger,\
                                      name=name+".parameter",\
                                      datatype=list(conf.parameter.datatypes),\
                                      logdir=logdir,\
                                      logfile=logfile,\
                                      datadir=datadir)
        try:
            parameter.transform = parameter.options.x.transform
            parameter.invtransform = parameter.options.x.invtransform
        except:
            parameter.transform = parameter.options.x.names
            parameter.invtransform = parameter.options.x.names

        # we now have access to parameter.x.state, parameter.x.sd etc
        # and possibly parameter.y.state etc.
        operators = []
        for (opname, op) in conf.operator.dict().iteritems():
            if opname != 'helper':
                #pdb.set_trace()
                exec('from eoldas_%s import %s' % (op.name, op.name))
                # make sure the data limits and x bounds are the same
                # ... inherit from parameter
                op.limits = parameter.options.limits
                if not 'datatypes' in op.dict():
                    op.datatypes = 'x'
                thisop = eval(op.name)(op,general,parameter=parameter,\
                                           logger=confs.logger,\
                                           name=name+".%s-%s"%(thisname,opname),\
                                           datatype=list(op.datatypes),\
                                           logdir=logdir,\
                                           logfile=logfile,\
                                           datadir=datadir)
                # load from parameter
                thisop.loader(parameter)
                operators.append(thisop)
                try:
                    thisop.transform = parameter.options.x.transform
                    thisop.invtransform = parameter.options.x.invtransform
                except:
                    thisop.transform = parameter.options.x.names
                    thisop.invtransform = parameter.options.x.names
                thisop.ploaderMask = np.in1d(parameter.x_meta.state,
                                             thisop.x_meta.state)
                try:
                    thisop.invtransform = np.array(
                        thisop.invtransform[thisop.ploaderMask])
                    thisop.transform = np.array(
                        thisop.transform[thisop.ploaderMask])
                except:
                    ww = thisop.ploaderMask
                    thisop.invtransform = np.array(thisop.transform)[ww]
                    thisop.transform = np.array(thisop.transform)[ww]
        # sort the loaders
        parameter.operators = operators
        self.confs.root.append(parameter)
        # Now we have set up the operators
        # try out the cost function
        if general.init_test:
            self.logger.info('testing cost function calls')
            J, J_prime = self.confs.root[0].cost()
            self.logger.info('done')
        self.confs.root = np.array(self.confs.root)
Esempio n. 11
0
    def reinit(self,options,names=None,datatype=None,limits=None,\
            bounds=None,control=None,location=None,env=None,header=None,\
               logdir=None,writers={},grid=False,logger=None,\
            datadir=None,logfile=None,name=None,info=[],readers=[],debug=None):
        '''
        Method to re-initialise the class instance
            
        The setup is on the whole controlled by the datatype which
            contains e.g. 'x'. This is used to set up the members
            self.x and self.y as SpecialVariables (see SpecialVariable
            in eoldas_SpecialVariable.py). There are some special attributes
            for datatypes starting with 'y'. These are assumed to be 
            observational data, which means that when they are read, the 
            data names associated with them are not limited to those in 
            self.names but rather set to whatever is read in in the data. This 
            is because the data names for observational data may be terms such 
            as waveband names etc that need special interpretation. Also,
            the default output format for observational data is 
            different to that of other data.  
            
        The elements self.state is a SpecialVariables which means 
            that they can be assigned various data types (see SpecialVariables)
            and loaded accordingly (e.g. if a filename is specified, this is 
            read in to the data structure. The SpecialVariables contain 'hidden'
            datasets, which here are mainly the 'control' and 'location'
            information. A SpecialVariable has two internal structures: `data`
            and `name`. The former is used to store data values (e.g. the 
            state values) and the latter to store associated metadata.
            For example, `control` is passed here e.g. as [`mask`,`vza`] and this
            gives the metadata that are stored in `name`. The actual values
            of the control data are stored in the `data` section. For
            location, we might be passed [`time`,`row`,`col`], so this is set
            in names.location, and the data.location contains the values
            of the location at each of these elements. For the actual state
            dataset, this is stored according to its name, so for `x` the values
            are stored in data.x and the associated data names in name.x.
        
        State datasets must represent at least the mean and standard deviation
            of a state for them to be of value in EOLDAS. TThe mean is accessed
            as e.g. self.state for the state dataset. 
            The sd is accessed can be accessed as self._state.sd if it 
            has been set. This reference can also be used to directly set 
            data associated with a SpecialVariable, e.g.
            self.Data.control = np.zeros([2,3])
            
            to represent 2 samples with 3 control variables. You can access name
            information similarly with
            
            print self.Name.control
            
            but this will generate a KeyError if the term has not been set. You
            can check it exists with:
            
            key = 'control'
            if key in self.Name:
                this = (self.Data[key],self.Name[key])
            
            To get e.g. a dictionary representation of a SpecialVariable
            you can use eg:
            
                self.Name.to_dict()
            
            to get the name dictionary, or
            
                thisdict = self._state.to_dict()
            
            to get the full representation, which then contains 'data' and
            'name' as well as some other information stored in the 
            SpecialVariable.
            
            You can similarly load them using e.g. 
            
                self.Data.update(
                        ParamStorage().from_dict(thisdict['data'])
                        combine=True)
        
        '''

        # set up a fakes dictionary from the data types
        self.set('datatype', datatype)
        self.set('fakes', {'state': '_state'})

        # first check that options is sensible
        self.__check_type(options, ParamStorage, fatal=True)
        self.options = options

        from eoldas_Lib import set_default_limits,\
                    check_limits_valid,quantize_location, sortopt

        nSpecial = 1
        if name == None:
            import time
            thistime = str(time.time())
            name = type(self).__name__
            name = "%s.%s" % (name, thistime)
        self.thisname = name
        self.options.thisname = str(name).replace(' ', '_')


        log_terms = {\
            'logfile':logfile or sortopt(self.options,'logfile',None),\
            'logdir':logdir or sortopt(self.options,'logdir',None),\
            'debug' : debug or sortopt(self.options,'debug',True)}

        self.datadir = datadir or sortopt(self.options, 'datadir', ["."])
        self.header = header or "EOLDAS pickle V1.0 - plewis"
        env = env or sortopt(self.options, 'env', None)
        names = names or sortopt(self.options, 'names', None)
        location = location or sortopt(self.options, 'location', ['time'])
        control = control or sortopt(self.options, 'control', [])
        limits = limits or sortopt(self.options,'limits',\
                        set_default_limits(np.array(location)))
        limits = limits or self.options.limits
        limits = np.array(check_limits_valid(limits))
        bounds = bounds or sortopt(self.options,'bounds',\
                    [[None,None]] * xlen(names))
        self.options.bounds = bounds

        self.headers = {'PARAMETERS-V2':"PARAMETERS-V2", \
            'PARAMETERS':"PARAMETERS", \
            'BRDF-UCL':'BRDF-UCL',\
            'BRDF': 'BRDF'}
        self.headers_2 = {'BRDF-UCL': 'location'}

        #  The ones pre-loaded are
        # self.read_functions = [self.read_pickle,self.read_numpy_fromfile]
        self._state = SpecialVariable(info=info,name=self.thisname,\
                                  readers=readers,datadir=self.datadir,\
                                  env=env,writers=writers,\
                                  header=self.header,\
                                  logger=logger,log_terms=log_terms,\
                                  simple=False)
        # self._state is where data are read into
        # but self.Data and self.Name are where we access them from
        self.grid = grid
        # this is so we can access this object from
        # inside a SpecialVariable
        self.state = np.array([0.])
        # a default data fmt output
        if datatype[0] == 'y':
            self.Name.fmt = 'BRDF'
            self.Name.state = np.array(['dummy'])
        else:
            self.Name.fmt = 'PARAMETERS'
            n_params = xlen(names)
            if not n_params:
                error_msg = \
                    "The field 'names' must be defined in options or"+ \
                    "passed directly to this method if you have the data type x"
                raise Exception(error_msg)
        self.Name.state = np.array(names)
        self.Name.location = np.array(location)
        self.Name.control = np.array(control)
        self.Name.header = self.header
        self.Name.bounds = np.array(bounds)
        self.Name.qlocation = np.array(limits)
        self.Name.datadir = datadir
        #
        # sort this object's name
        # sort logging
        self.logger = sortlog(self,
                              log_terms['logfile'],
                              logger,
                              name=self.thisname,
                              logdir=log_terms['logdir'],
                              debug=log_terms['debug'])
        self.logger.info('Initialising %s' % type(self).__name__)