def log(self,
this=None,
logger=None,
name=None,
supername='',
logfile="log.dat",
n=0,
logdir="logs",
debug=True,
print_helper=False):
'''
Dump the whole contents to the a log
Useful for inspecting data
If self.logger is set, this is the log that is used, otherwise
one is set up from logfile (and possibly logdir).
In this case, name is used as the identifier in the log. You
have to set name to enable log initialisation.
If print_helper is True, then print_helper fields are logged (default False)
'''
from eoldas_Lib import sortlog
self.logger = sortlog(self,
logfile,
logger,
name=name,
logdir=logdir,
debug=debug)
if this == None:
this = self
try:
self.logger.info("**" * 20)
self.logger.info("logging parameters ...")
self.logger.info("**" * 20)
except:
pass
for (k, v) in this.iteritems():
if supername != '':
thisname = "%s.%s" % (supername, str(k))
else:
thisname = str(k)
strk = str(k)
doit = True
if strk.replace('__', '') != strk or str(k) == 'helper' or str(
k)[:5] == "help_":
doit = False
if doit or print_helper:
try:
self.logger.info("%s = %s" % (thisname, str(v)))
except:
pass
if type(v) == ParamStorage and k != 'logger':
self.log(this=this[k],
name=None,
n=n + 2,
supername=thisname,
print_helper=print_helper)
def __init__(self,options,name=None,loaders=[],logfile=None,\
logdir=None,datadir=['.','~/.eoldas'],logger=None\
,env="EOLDAS",fatal=False):
'''
Parse the configuration file(s) referred to in the (string or list)
"options".
Parameters:
options : The class is initialised with "options",
which may be of type ParamStorage
or a list or a string.
"options" may also be of type ParamStorage,
in which case it is simply loaded here.
Options:
log_name : name of log. If none is specified,
logging goes to stdout.
log : boolean, specifying whether or not to do
logging.
datadir : a list of strings, specifying where to
look for configuration files.
env : an environment variable that may contain
more places to search for configuration files
(after datadir) (default "EOLDAS").
fatal : specify whether not finding a configuration
file should be fatal or not (default False)
See type_convert() for some rules on defining a configration file.
'''
from eoldas_Lib import sortlog
if name == None:
import time
thistime = str(time.time())
name = type(self).__name__
name = "%s.%s" % (name, thistime)
self.thisname = name
self.logdir = logdir
self.logfile = logfile
self.storelog = ""
self.datadir = datadir
self.env = env
self.fatal = fatal
self.logger = sortlog(self,
self.logfile,
logger,
name=self.thisname,
logdir=self.logdir,
debug=True)
self.configs = []
self.infos = []
self.loaders = loaders
if type(options) == ParamStorage:
self.options.update(options, combine=True)
if type(options) == list or type(options) == str:
self.read_conf_file(options)
def __init__(self,options,name=None,loaders=[],logfile=None,\
logdir=None,datadir=['.','~/.eoldas'],logger=None\
,env="EOLDAS",fatal=False):
'''
Parse the configuration file(s) referred to in the (string or list)
"options".
Parameters:
options : The class is initialised with "options",
which may be of type ParamStorage
or a list or a string.
"options" may also be of type ParamStorage,
in which case it is simply loaded here.
Options:
log_name : name of log. If none is specified,
logging goes to stdout.
log : boolean, specifying whether or not to do
logging.
datadir : a list of strings, specifying where to
look for configuration files.
env : an environment variable that may contain
more places to search for configuration files
(after datadir) (default "EOLDAS").
fatal : specify whether not finding a configuration
file should be fatal or not (default False)
See type_convert() for some rules on defining a configration file.
'''
from eoldas_Lib import sortlog
if name == None:
import time
thistime = str(time.time())
name = type(self).__name__
name = "%s.%s" % (name,thistime)
self.thisname = name
self.logdir = logdir
self.logfile = logfile
self.storelog = ""
self.datadir=datadir
self.env=env
self.fatal=fatal
self.logger = sortlog(self,self.logfile,logger,name=self.thisname,logdir=self.logdir,debug=True)
self.configs = []
self.infos = []
self.loaders = loaders
if type(options) == ParamStorage:
self.options.update(options,combine=True)
if type(options)== list or type(options) == str:
self.read_conf_file(options)
def log(self,this=None,logger=None,name=None,supername='',logfile="log.dat",n=0,logdir="logs",debug=True,print_helper=False):
'''
Dump the whole contents to the a log
Useful for inspecting data
If self.logger is set, this is the log that is used, otherwise
one is set up from logfile (and possibly logdir).
In this case, name is used as the identifier in the log. You
have to set name to enable log initialisation.
If print_helper is True, then print_helper fields are logged (default False)
'''
from eoldas_Lib import sortlog
self.logger = sortlog(self,logfile,logger,name=name,logdir=logdir,debug=debug)
if this == None:
this = self
try:
self.logger.info("**"*20)
self.logger.info("logging parameters ...")
self.logger.info("**"*20)
except:
pass
for ( k, v ) in this.iteritems():
if supername != '':
thisname = "%s.%s" % (supername,str(k))
else:
thisname = str(k)
strk = str(k)
doit = True
if strk.replace('__','') != strk or str(k) == 'helper' or str(k)[:5] == "help_":
doit = False
if doit or print_helper:
try:
self.logger.info("%s = %s" % (thisname,str(v)))
except:
pass
if type(v) == ParamStorage and k != 'logger':
self.log(this=this[k],name=None,n=n+2,supername=thisname,print_helper=print_helper)
def __init__(self,args,name=None,general=None,log=False,logger=None,outdir=".",getopdir=False,parse=True):
"""
Initialise parser class.
This sets up the class defaults.
Options:
general=general: this over-rides and defaults with values
set in parser general can be of the form:
1. class ParamStorage (i.e. the same form
as self.general)
2. a command line list (where the first
item in the list is ignored
3. a string containing a set of command line general
See self.parse() for more details on general
2 and 3 as these simply make a call to tha
method.
log=True If log is set to True, then logging starts
when this class is instanced.
Note that the logfile and logdir might
change if subsequent calls to Parser.parse()
are made
"""
if type(args) == str:
args = args.split()
self.dolog = log
self.log = log
self.name = args[0]
self.args = args[1:]
self.fullargs = args
self.store_fullargs = args
if name == None:
import time
thistime = str(time.time())
name = type(self).__name__
name = "%s.%s" % (name,thistime)
self.thisname = name
# find the following flags:
# --conf | -c : conf
# --datadir : datadir
datadir = [".","~/.eoldas",sys.path[0]+'/../bin',sys.path[0]+'/../confs',\
sys.path[0]+'/../system_confs',sys.path[0]+'/../eoldaslib']
conf = "default.conf"
logfile = None
logdir = "."
self.top = ParamStorage ()
self.top.general = ParamStorage ()
self.top.general.__helper__ = ParamStorage ()
self.top.general.__default__ = ParamStorage ()
self.top.general.__extras__ = ParamStorage ()
self.top.general.conf = []
for i in xrange(len(self.args)):
theseargs = self.args[i].split('=')
if theseargs[0] == "--conf":
conf = theseargs[1]
self.top.general.conf.append(conf)
elif theseargs[0][0:2] == "-c":
if len(theseargs) > 2:
conf = theseargs[0][2:]
else:
conf = self.args[i+1]
self.top.general.conf.append(conf)
elif theseargs[0] == "--datadir":
datadir1 = theseargs[1].replace('[','').\
replace(']','').split()
[datadir1.append(datadir[i]) for i in \
xrange(len(datadir))]
datadir = datadir1
elif theseargs[0] == "--logfile":
logfile= theseargs[1]
elif theseargs[0] == "--logdir":
logdir = theseargs[1]
elif theseargs[0] == "--outdir":
outdir = theseargs[1]
if self.top.general.conf == []:
self.top.general.conf = conf
if logfile == None:
logfile = conf.replace('conf','log')
self.top.general.here = os.getcwd()
self.top.general.datadir = datadir
self.top.general.logfile = logfile
self.top.general.logdir = logdir
self.top.general.outdir = outdir
# add here to datadir
# in addition to '.' to take account of the change of directory
self.top.general.datadir = self.__add_here_to_datadir(\
self.top.general.here,self.top.general.datadir)
self.top.general.datadir = self.__add_here_to_datadir(\
self.top.general.outdir,self.top.general.datadir)
# cd to where the output is to be
self.__cd(self.top.general.outdir)
# set up the default command line options
self.default_loader()
# update with anything passed here
if general and type(general) == ParamStorage: self.top.update(\
self.__unload(general),combine=True)
# read the conf files to get any cmd line options
self.logger = sortlog(self,self.top.general.logfile,logger,name=self.thisname,\
logdir=self.top.general.logdir)
self.config = ConfFile(self.top.general.conf,name=self.thisname+'.config',\
loaders=self.loaders,datadir=self.top.\
general.datadir,logger=self.logger,logdir=self.top.general.logdir,\
logfile=self.top.general.logfile)
if len(self.config.configs) == 0:
this = "Warning: Nothing doing ... you haven't set any configuration",\
self.config.storelog
try:
self.logger(this)
except:
print "Called with args:"
print "eoldas",self.args
pass
raise Exception(this)
# now loaders contains all of the defaults set here
# plus those from the config (config opver-rides defaults here)
self.loaders = self.config.loaders
# now convert loaders into parser information
self.parseLoader(self.loaders)
self.parse(self.fullargs)
if general and type(general) == ParamStorage:
self.top.update(self.__unload(general),combine=True)
if general and type(general) == str:
self.parse(general.split())
if general and type(general) == list:
self.parse(general)
# now update the info in self.config
for i in self.config.infos:
i.update(self.top,combine=True)
# so now all terms in self.config.infos
# contain information from the config file, updated by
# the cmd line
i.logger = self.logger
i.log()
# move the information up a level
self.infos = self.config.infos
self.configs = self.config.configs
self.config_log = self.config.storelog
#if getopdir:
# self.sortnames()
self.config.loglist(self.top)
#del(self.config.infos)
#del(self.config.configs)
#del(self.config)
self.__cd(self.top.general.here)
def init(self,info=[],name=None,readers=[],log_terms={},\
datadir=None,env=None,\
header=None,writers={},\
simple=False,logger=None):
'''
Initialise information in a SpecialVariable instance.
Options:
info : Information tthat can be passed through to reader
methods (a list).
thisname : a name to use to identify this instance in any
logging. By default this is None. If thisname is set
to True, then logging is to stdout.
readers : A list of reader methods that are pre-pended to
those already contained in the class.
log_terms : A dictionary of log options. By default
{'logfile':None,'logdir':'log','debug':True}
If thisname is set, and logfile specified, then
logs are logged to that file. If thisname is set
to True, then logging is to stdout.
datadir : A list of directories to search for data files
to interpret if the SpecialVariable is set to a
string.
env : An environment variable that can be used to extend
the datadir variable.
header : A header string to use to identify pickle files.
By default, this is set to
"EOLDAS -- plewis -- UCL -- V0.1"
simple : A flag to swicth off the 'complicated'
interpretation methods, i.e. just set and return
variables literally, do not try to interpret them.
'''
self.set('simple',True)
if name == None:
import time
thistime = str(time.time())
name = type(self).__name__
name = "%s.%s" % (name,thistime)
self.thisname = name
# this is where we will put any data
self.data = ParamStorage()
self.name = ParamStorage()
self.info = info
self.datadir = datadir or ['.']
self.env = env
init_read_write(self,header,readers,writers)
# sort logging and log if thisname != None
self.log_terms = {'logfile':None,'logdir':'log','debug':True}
# override logging info
for (key,value) in log_terms.iteritems():
self.log_terms[key] = value
self.logger= sortlog(self,self.log_terms['logfile'],logger,name=self.thisname,\
logdir=self.log_terms['logdir'],\
debug=self.log_terms['debug'])
self.simple = simple
def reinit(self,options,names=None,datatype=None,limits=None,\
bounds=None,control=None,location=None,env=None,header=None,\
logdir=None,writers={},grid=False,logger=None,\
datadir=None,logfile=None,name=None,info=[],readers=[],debug=None):
'''
Method to re-initialise the class instance
The setup is on the whole controlled by the datatype which
contains e.g. 'x'. This is used to set up the members
self.x and self.y as SpecialVariables (see SpecialVariable
in eoldas_SpecialVariable.py). There are some special attributes
for datatypes starting with 'y'. These are assumed to be
observational data, which means that when they are read, the
data names associated with them are not limited to those in
self.names but rather set to whatever is read in in the data. This
is because the data names for observational data may be terms such
as waveband names etc that need special interpretation. Also,
the default output format for observational data is
different to that of other data.
The elements self.state is a SpecialVariables which means
that they can be assigned various data types (see SpecialVariables)
and loaded accordingly (e.g. if a filename is specified, this is
read in to the data structure. The SpecialVariables contain 'hidden'
datasets, which here are mainly the 'control' and 'location'
information. A SpecialVariable has two internal structures: `data`
and `name`. The former is used to store data values (e.g. the
state values) and the latter to store associated metadata.
For example, `control` is passed here e.g. as [`mask`,`vza`] and this
gives the metadata that are stored in `name`. The actual values
of the control data are stored in the `data` section. For
location, we might be passed [`time`,`row`,`col`], so this is set
in names.location, and the data.location contains the values
of the location at each of these elements. For the actual state
dataset, this is stored according to its name, so for `x` the values
are stored in data.x and the associated data names in name.x.
State datasets must represent at least the mean and standard deviation
of a state for them to be of value in EOLDAS. TThe mean is accessed
as e.g. self.state for the state dataset.
The sd is accessed can be accessed as self._state.sd if it
has been set. This reference can also be used to directly set
data associated with a SpecialVariable, e.g.
self.Data.control = np.zeros([2,3])
to represent 2 samples with 3 control variables. You can access name
information similarly with
print self.Name.control
but this will generate a KeyError if the term has not been set. You
can check it exists with:
key = 'control'
if key in self.Name:
this = (self.Data[key],self.Name[key])
To get e.g. a dictionary representation of a SpecialVariable
you can use eg:
self.Name.to_dict()
to get the name dictionary, or
thisdict = self._state.to_dict()
to get the full representation, which then contains 'data' and
'name' as well as some other information stored in the
SpecialVariable.
You can similarly load them using e.g.
self.Data.update(
ParamStorage().from_dict(thisdict['data'])
combine=True)
'''
# set up a fakes dictionary from the data types
self.set('datatype',datatype)
self.set('fakes', {'state':'_state'})
# first check that options is sensible
self.__check_type(options,ParamStorage,fatal=True)
self.options = options
from eoldas_Lib import set_default_limits,\
check_limits_valid,quantize_location, sortopt
nSpecial = 1
if name == None:
import time
thistime = str(time.time())
name = type(self).__name__
name = "%s.%s" % (name,thistime)
self.thisname = name
self.options.thisname = str(name).replace(' ','_')
log_terms = {\
'logfile':logfile or sortopt(self.options,'logfile',None),\
'logdir':logdir or sortopt(self.options,'logdir',None),\
'debug' : debug or sortopt(self.options,'debug',True)}
self.datadir = datadir or sortopt(self.options,'datadir',["."])
self.header = header or "EOLDAS pickle V1.0 - plewis"
env = env or sortopt(self.options,'env',None)
names = names or sortopt(self.options,'names',None)
location = location or sortopt(self.options,'location',['time'])
control = control or sortopt(self.options,'control',[])
limits = limits or sortopt(self.options,'limits',\
set_default_limits(np.array(location)))
limits = limits or self.options.limits
limits = np.array(check_limits_valid(limits))
bounds = bounds or sortopt(self.options,'bounds',\
[[None,None]] * xlen(names))
self.options.bounds = bounds
self.headers = {'PARAMETERS-V2':"PARAMETERS-V2", \
'PARAMETERS':"PARAMETERS", \
'BRDF-UCL':'BRDF-UCL',\
'BRDF': 'BRDF'}
self.headers_2 = {'BRDF-UCL':'location'}
# The ones pre-loaded are
# self.read_functions = [self.read_pickle,self.read_numpy_fromfile]
self._state = SpecialVariable(info=info,name=self.thisname,\
readers=readers,datadir=self.datadir,\
env=env,writers=writers,\
header=self.header,\
logger=logger,log_terms=log_terms,\
simple=False)
# self._state is where data are read into
# but self.Data and self.Name are where we access them from
self.grid=grid
# this is so we can access this object from
# inside a SpecialVariable
self.state = np.array([0.])
# a default data fmt output
if datatype[0] == 'y':
self.Name.fmt = 'BRDF'
self.Name.state = np.array(['dummy'])
else:
self.Name.fmt = 'PARAMETERS'
n_params = xlen(names)
if not n_params:
error_msg = \
"The field 'names' must be defined in options or"+ \
"passed directly to this method if you have the data type x"
raise Exception(error_msg)
self.Name.state = np.array(names)
self.Name.location = np.array(location)
self.Name.control = np.array(control)
self.Name.header = self.header
self.Name.bounds = np.array(bounds)
self.Name.qlocation = np.array(limits)
self.Name.datadir = datadir
#
# sort this object's name
# sort logging
self.logger = sortlog(self,log_terms['logfile'],logger,name=self.thisname,
logdir=log_terms['logdir'],debug=log_terms['debug'])
self.logger.info('Initialising %s' % type(self).__name__)
def __init__(self,
args,
name=None,
general=None,
log=False,
logger=None,
outdir=".",
getopdir=False,
parse=True):
"""
Initialise parser class.
This sets up the class defaults.
Options:
general=general: this over-rides and defaults with values
set in parser general can be of the form:
1. class ParamStorage (i.e. the same form
as self.general)
2. a command line list (where the first
item in the list is ignored
3. a string containing a set of command line general
See self.parse() for more details on general
2 and 3 as these simply make a call to tha
method.
log=True If log is set to True, then logging starts
when this class is instanced.
Note that the logfile and logdir might
change if subsequent calls to Parser.parse()
are made
"""
if type(args) == str:
args = args.split()
self.dolog = log
self.log = log
self.name = args[0]
self.args = args[1:]
self.fullargs = args
self.store_fullargs = args
if name == None:
import time
thistime = str(time.time())
name = type(self).__name__
name = "%s.%s" % (name, thistime)
self.thisname = name
# find the following flags:
# --conf | -c : conf
# --datadir : datadir
datadir = [".","~/.eoldas",sys.path[0]+'/../bin',sys.path[0]+'/../confs',\
sys.path[0]+'/../system_confs',sys.path[0]+'/../eoldaslib']
conf = "default.conf"
logfile = None
logdir = "."
self.top = ParamStorage()
self.top.general = ParamStorage()
self.top.general.__helper__ = ParamStorage()
self.top.general.__default__ = ParamStorage()
self.top.general.__extras__ = ParamStorage()
self.top.general.conf = []
for i in xrange(len(self.args)):
theseargs = self.args[i].split('=')
if theseargs[0] == "--conf":
conf = theseargs[1]
self.top.general.conf.append(conf)
elif theseargs[0][0:2] == "-c":
if len(theseargs) > 2:
conf = theseargs[0][2:]
else:
conf = self.args[i + 1]
self.top.general.conf.append(conf)
elif theseargs[0] == "--datadir":
datadir1 = theseargs[1].replace('[','').\
replace(']','').split()
[datadir1.append(datadir[i]) for i in \
xrange(len(datadir))]
datadir = datadir1
elif theseargs[0] == "--logfile":
logfile = theseargs[1]
elif theseargs[0] == "--logdir":
logdir = theseargs[1]
elif theseargs[0] == "--outdir":
outdir = theseargs[1]
if self.top.general.conf == []:
self.top.general.conf = conf
if logfile == None:
logfile = conf.replace('conf', 'log')
self.top.general.here = os.getcwd()
self.top.general.datadir = datadir
self.top.general.logfile = logfile
self.top.general.logdir = logdir
self.top.general.outdir = outdir
# add here to datadir
# in addition to '.' to take account of the change of directory
self.top.general.datadir = self.__add_here_to_datadir(\
self.top.general.here,self.top.general.datadir)
self.top.general.datadir = self.__add_here_to_datadir(\
self.top.general.outdir,self.top.general.datadir)
# cd to where the output is to be
self.__cd(self.top.general.outdir)
# set up the default command line options
self.default_loader()
# update with anything passed here
if general and type(general) == ParamStorage: self.top.update(\
self.__unload(general),combine=True)
# read the conf files to get any cmd line options
self.logger = sortlog(self,self.top.general.logfile,logger,name=self.thisname,\
logdir=self.top.general.logdir)
self.config = ConfFile(self.top.general.conf,name=self.thisname+'.config',\
loaders=self.loaders,datadir=self.top.\
general.datadir,logger=self.logger,logdir=self.top.general.logdir,\
logfile=self.top.general.logfile)
if len(self.config.configs) == 0:
this = "Warning: Nothing doing ... you haven't set any configuration",\
self.config.storelog
try:
self.logger(this)
except:
print "Called with args:"
print "eoldas", self.args
pass
raise Exception(this)
# now loaders contains all of the defaults set here
# plus those from the config (config opver-rides defaults here)
self.loaders = self.config.loaders
# now convert loaders into parser information
self.parseLoader(self.loaders)
self.parse(self.fullargs)
if general and type(general) == ParamStorage:
self.top.update(self.__unload(general), combine=True)
if general and type(general) == str:
self.parse(general.split())
if general and type(general) == list:
self.parse(general)
# now update the info in self.config
for i in self.config.infos:
i.update(self.top, combine=True)
# so now all terms in self.config.infos
# contain information from the config file, updated by
# the cmd line
i.logger = self.logger
i.log()
# move the information up a level
self.infos = self.config.infos
self.configs = self.config.configs
self.config_log = self.config.storelog
#if getopdir:
# self.sortnames()
self.config.loglist(self.top)
#del(self.config.infos)
#del(self.config.configs)
#del(self.config)
self.__cd(self.top.general.here)
def __init__(self,confs,logger=None,logfile=None,thisname=None,name=None,datadir=None,logdir=None):
'''
Initialise the solver.
This does the following:
1. Read configuration file(s)
2. Load operators
3. Test the call to the cost function
There can be multiple groups of configuration files, so
self.confs, that holds the core information setup here
can contain multiple configurations.
The number of configurations is len(confs.infos) and
the ith configuration is conf = self.confs.infos[i].
Various loggers are available throughout the classes
used, but the top level logger is self.confs.logger,
so you can log with e.g.
self.confs.logger.info('this is some info')
The root operator is stored in self.confs.root[i]
for the ith configuration, so the basic call to the cost
function is:
J,J_prime = self.confs[i].parameter.cost(None)
'''
from eoldas_ConfFile import ConfFile
from eoldas_Lib import sortopt
name = name or thisname
if name == None:
import time
thistime = str(time.time())
name = type(self).__name__
name = "%s.%s" % (name,thistime)
self.thisname = name
self.confs = confs
self.top = sortopt(self,'top',ParamStorage())
self.top.general = sortopt(self.top,'general',ParamStorage())
thisname = sortopt(self.top.general,'name',thisname or self.thisname)
logfile = sortopt(self.top.general,'logfile',logfile or 'log.dat')
logdir = sortopt(self.top.general,'logdir',logfile or 'logs')
datadir = sortopt(self.top.general,'datadir',datadir or ['.'])
self.logger = sortlog(self,logfile,logger or self.confs.logger,\
name=self.thisname,logdir=logdir,debug=True)
n_configs = len(self.confs.infos)
self.confs.root = []
self.have_unc = False
try:
logdir = logdir
except:
logdir = self.top.general.logdir
# first set up parameter
conf = confs.infos[0]
general = conf.general
op = conf.parameter
if not 'parameter' in conf.dict():
raise Exception('No parameter field found in %s item %d'%\
(conf.__doc__,0))
general.is_spectral = sortopt(general,'is_spectral',True)
if not general.is_spectral:
sort_non_spectral_model(op,conf.operator,logger=confs.logger)
general.init_test = sortopt(general,'init_test',False)
confs.logger.info('loading parameter state')
conf.parameter.name = 'Operator'
parameter = eval(op.name)(op,general,\
parameter=None,\
logger=confs.logger,\
name=name+".parameter",\
datatype=list(conf.parameter.datatypes),\
logdir=logdir,\
logfile=logfile,\
datadir=datadir)
try:
parameter.transform = parameter.options.x.transform
parameter.invtransform = parameter.options.x.invtransform
except:
parameter.transform = parameter.options.x.names
parameter.invtransform = parameter.options.x.names
# we now have access to parameter.x.state, parameter.x.sd etc
# and possibly parameter.y.state etc.
operators = []
for (opname,op) in conf.operator.dict().iteritems():
if opname != 'helper':
#pdb.set_trace()
exec('from eoldas_%s import %s'%(op.name,op.name))
# make sure the data limits and x bounds are the same
# ... inherit from parameter
op.limits = parameter.options.limits
if not 'datatypes' in op.dict():
op.datatypes = 'x'
thisop = eval(op.name)(op,general,parameter=parameter,\
logger=confs.logger,\
name=name+".%s-%s"%(thisname,opname),\
datatype=list(op.datatypes),\
logdir=logdir,\
logfile=logfile,\
datadir=datadir)
# load from parameter
thisop.loader(parameter)
operators.append(thisop)
try:
thisop.transform = parameter.options.x.transform
thisop.invtransform = parameter.options.x.invtransform
except:
thisop.transform = parameter.options.x.names
thisop.invtransform = parameter.options.x.names
thisop.ploaderMask = np.in1d(parameter.x_meta.state,thisop.x_meta.state)
try:
thisop.invtransform = np.array(thisop.invtransform[thisop.ploaderMask])
thisop.transform = np.array(thisop.transform[thisop.ploaderMask])
except:
ww = thisop.ploaderMask
thisop.invtransform = np.array(thisop.transform)[ww]
thisop.transform = np.array(thisop.transform)[ww]
# sort the loaders
parameter.operators = operators
self.confs.root.append(parameter)
# Now we have set up the operators
# try out the cost function
if general.init_test:
self.logger.info('testing cost function calls')
J,J_prime = self.confs.root[0].cost()
self.logger.info('done')
self.confs.root = np.array(self.confs.root)
def __init__(self,
confs,
logger=None,
logfile=None,
thisname=None,
name=None,
datadir=None,
logdir=None):
'''
Initialise the solver.
This does the following:
1. Read configuration file(s)
2. Load operators
3. Test the call to the cost function
There can be multiple groups of configuration files, so
self.confs, that holds the core information setup here
can contain multiple configurations.
The number of configurations is len(confs.infos) and
the ith configuration is conf = self.confs.infos[i].
Various loggers are available throughout the classes
used, but the top level logger is self.confs.logger,
so you can log with e.g.
self.confs.logger.info('this is some info')
The root operator is stored in self.confs.root[i]
for the ith configuration, so the basic call to the cost
function is:
J,J_prime = self.confs[i].parameter.cost(None)
'''
from eoldas_ConfFile import ConfFile
from eoldas_Lib import sortopt
name = name or thisname
if name == None:
import time
thistime = str(time.time())
name = type(self).__name__
name = "%s.%s" % (name, thistime)
self.thisname = name
self.confs = confs
self.top = sortopt(self, 'top', ParamStorage())
self.top.general = sortopt(self.top, 'general', ParamStorage())
thisname = sortopt(self.top.general, 'name', thisname or self.thisname)
logfile = sortopt(self.top.general, 'logfile', logfile or 'log.dat')
logdir = sortopt(self.top.general, 'logdir', logfile or 'logs')
datadir = sortopt(self.top.general, 'datadir', datadir or ['.'])
self.logger = sortlog(self,logfile,logger or self.confs.logger,\
name=self.thisname,logdir=logdir,debug=True)
n_configs = len(self.confs.infos)
self.confs.root = []
self.have_unc = False
try:
logdir = logdir
except:
logdir = self.top.general.logdir
# first set up parameter
conf = confs.infos[0]
general = conf.general
op = conf.parameter
if not 'parameter' in conf.dict():
raise Exception('No parameter field found in %s item %d'%\
(conf.__doc__,0))
general.is_spectral = sortopt(general, 'is_spectral', True)
if not general.is_spectral:
sort_non_spectral_model(op, conf.operator, logger=confs.logger)
general.init_test = sortopt(general, 'init_test', False)
confs.logger.info('loading parameter state')
conf.parameter.name = 'Operator'
parameter = eval(op.name)(op,general,\
parameter=None,\
logger=confs.logger,\
name=name+".parameter",\
datatype=list(conf.parameter.datatypes),\
logdir=logdir,\
logfile=logfile,\
datadir=datadir)
try:
parameter.transform = parameter.options.x.transform
parameter.invtransform = parameter.options.x.invtransform
except:
parameter.transform = parameter.options.x.names
parameter.invtransform = parameter.options.x.names
# we now have access to parameter.x.state, parameter.x.sd etc
# and possibly parameter.y.state etc.
operators = []
for (opname, op) in conf.operator.dict().iteritems():
if opname != 'helper':
#pdb.set_trace()
exec('from eoldas_%s import %s' % (op.name, op.name))
# make sure the data limits and x bounds are the same
# ... inherit from parameter
op.limits = parameter.options.limits
if not 'datatypes' in op.dict():
op.datatypes = 'x'
thisop = eval(op.name)(op,general,parameter=parameter,\
logger=confs.logger,\
name=name+".%s-%s"%(thisname,opname),\
datatype=list(op.datatypes),\
logdir=logdir,\
logfile=logfile,\
datadir=datadir)
# load from parameter
thisop.loader(parameter)
operators.append(thisop)
try:
thisop.transform = parameter.options.x.transform
thisop.invtransform = parameter.options.x.invtransform
except:
thisop.transform = parameter.options.x.names
thisop.invtransform = parameter.options.x.names
thisop.ploaderMask = np.in1d(parameter.x_meta.state,
thisop.x_meta.state)
try:
thisop.invtransform = np.array(
thisop.invtransform[thisop.ploaderMask])
thisop.transform = np.array(
thisop.transform[thisop.ploaderMask])
except:
ww = thisop.ploaderMask
thisop.invtransform = np.array(thisop.transform)[ww]
thisop.transform = np.array(thisop.transform)[ww]
# sort the loaders
parameter.operators = operators
self.confs.root.append(parameter)
# Now we have set up the operators
# try out the cost function
if general.init_test:
self.logger.info('testing cost function calls')
J, J_prime = self.confs.root[0].cost()
self.logger.info('done')
self.confs.root = np.array(self.confs.root)
def reinit(self,options,names=None,datatype=None,limits=None,\
bounds=None,control=None,location=None,env=None,header=None,\
logdir=None,writers={},grid=False,logger=None,\
datadir=None,logfile=None,name=None,info=[],readers=[],debug=None):
'''
Method to re-initialise the class instance
The setup is on the whole controlled by the datatype which
contains e.g. 'x'. This is used to set up the members
self.x and self.y as SpecialVariables (see SpecialVariable
in eoldas_SpecialVariable.py). There are some special attributes
for datatypes starting with 'y'. These are assumed to be
observational data, which means that when they are read, the
data names associated with them are not limited to those in
self.names but rather set to whatever is read in in the data. This
is because the data names for observational data may be terms such
as waveband names etc that need special interpretation. Also,
the default output format for observational data is
different to that of other data.
The elements self.state is a SpecialVariables which means
that they can be assigned various data types (see SpecialVariables)
and loaded accordingly (e.g. if a filename is specified, this is
read in to the data structure. The SpecialVariables contain 'hidden'
datasets, which here are mainly the 'control' and 'location'
information. A SpecialVariable has two internal structures: `data`
and `name`. The former is used to store data values (e.g. the
state values) and the latter to store associated metadata.
For example, `control` is passed here e.g. as [`mask`,`vza`] and this
gives the metadata that are stored in `name`. The actual values
of the control data are stored in the `data` section. For
location, we might be passed [`time`,`row`,`col`], so this is set
in names.location, and the data.location contains the values
of the location at each of these elements. For the actual state
dataset, this is stored according to its name, so for `x` the values
are stored in data.x and the associated data names in name.x.
State datasets must represent at least the mean and standard deviation
of a state for them to be of value in EOLDAS. TThe mean is accessed
as e.g. self.state for the state dataset.
The sd is accessed can be accessed as self._state.sd if it
has been set. This reference can also be used to directly set
data associated with a SpecialVariable, e.g.
self.Data.control = np.zeros([2,3])
to represent 2 samples with 3 control variables. You can access name
information similarly with
print self.Name.control
but this will generate a KeyError if the term has not been set. You
can check it exists with:
key = 'control'
if key in self.Name:
this = (self.Data[key],self.Name[key])
To get e.g. a dictionary representation of a SpecialVariable
you can use eg:
self.Name.to_dict()
to get the name dictionary, or
thisdict = self._state.to_dict()
to get the full representation, which then contains 'data' and
'name' as well as some other information stored in the
SpecialVariable.
You can similarly load them using e.g.
self.Data.update(
ParamStorage().from_dict(thisdict['data'])
combine=True)
'''
# set up a fakes dictionary from the data types
self.set('datatype', datatype)
self.set('fakes', {'state': '_state'})
# first check that options is sensible
self.__check_type(options, ParamStorage, fatal=True)
self.options = options
from eoldas_Lib import set_default_limits,\
check_limits_valid,quantize_location, sortopt
nSpecial = 1
if name == None:
import time
thistime = str(time.time())
name = type(self).__name__
name = "%s.%s" % (name, thistime)
self.thisname = name
self.options.thisname = str(name).replace(' ', '_')
log_terms = {\
'logfile':logfile or sortopt(self.options,'logfile',None),\
'logdir':logdir or sortopt(self.options,'logdir',None),\
'debug' : debug or sortopt(self.options,'debug',True)}
self.datadir = datadir or sortopt(self.options, 'datadir', ["."])
self.header = header or "EOLDAS pickle V1.0 - plewis"
env = env or sortopt(self.options, 'env', None)
names = names or sortopt(self.options, 'names', None)
location = location or sortopt(self.options, 'location', ['time'])
control = control or sortopt(self.options, 'control', [])
limits = limits or sortopt(self.options,'limits',\
set_default_limits(np.array(location)))
limits = limits or self.options.limits
limits = np.array(check_limits_valid(limits))
bounds = bounds or sortopt(self.options,'bounds',\
[[None,None]] * xlen(names))
self.options.bounds = bounds
self.headers = {'PARAMETERS-V2':"PARAMETERS-V2", \
'PARAMETERS':"PARAMETERS", \
'BRDF-UCL':'BRDF-UCL',\
'BRDF': 'BRDF'}
self.headers_2 = {'BRDF-UCL': 'location'}
# The ones pre-loaded are
# self.read_functions = [self.read_pickle,self.read_numpy_fromfile]
self._state = SpecialVariable(info=info,name=self.thisname,\
readers=readers,datadir=self.datadir,\
env=env,writers=writers,\
header=self.header,\
logger=logger,log_terms=log_terms,\
simple=False)
# self._state is where data are read into
# but self.Data and self.Name are where we access them from
self.grid = grid
# this is so we can access this object from
# inside a SpecialVariable
self.state = np.array([0.])
# a default data fmt output
if datatype[0] == 'y':
self.Name.fmt = 'BRDF'
self.Name.state = np.array(['dummy'])
else:
self.Name.fmt = 'PARAMETERS'
n_params = xlen(names)
if not n_params:
error_msg = \
"The field 'names' must be defined in options or"+ \
"passed directly to this method if you have the data type x"
raise Exception(error_msg)
self.Name.state = np.array(names)
self.Name.location = np.array(location)
self.Name.control = np.array(control)
self.Name.header = self.header
self.Name.bounds = np.array(bounds)
self.Name.qlocation = np.array(limits)
self.Name.datadir = datadir
#
# sort this object's name
# sort logging
self.logger = sortlog(self,
log_terms['logfile'],
logger,
name=self.thisname,
logdir=log_terms['logdir'],
debug=log_terms['debug'])
self.logger.info('Initialising %s' % type(self).__name__)
