def preload_prepare(self):
'''
Here , we use preload_prepare to make sure
the x & any y data are gridded for this
operator. This greatly simplifies the
application of the differential operator.
This method is called before any data are loaded,
so ensures they are loaded as a grid.
'''
from eoldas_Lib import sortopt
for i in np.array(self.options.datatypes).flatten():
# mimic setting the apply_grid flag in options
if self.dict().has_key('%s_state' % i):
self['%s_state' % i].options[i].apply_grid = True
self.novar = sortopt(self, 'novar', False)
self.gamma_col = sortopt(self, 'gamma_col', None)
self.beenHere = False
def preload_prepare(self):
'''
Here , we use preload_prepare to make sure
the x & any y data are gridded for this
operator. This greatly simplifies the
application of the differential operator.
This method is called before any data are loaded,
so ensures they are loaded as a grid.
'''
from eoldas_Lib import sortopt
for i in np.array(self.options.datatypes).flatten():
# mimic setting the apply_grid flag in options
if self.dict().has_key('%s_state'%i):
self['%s_state'%i].options[i].apply_grid = True
self.novar = sortopt(self,'novar',False)
self.gamma_col = sortopt(self,'gamma_col',None)
self.beenHere =False
def sortMask(self):
'''
Sort masks for loading and unloading data
'''
self.root.options.solve = \
sortopt(self.root.options,\
'solve',np.array([1]*len(self.root.x_meta.state)))
self.solve = np.array(self.root.options.solve).copy()
# set up masks for loading and unloading
self.mask1 = np.zeros_like(self.root.x.state).astype(bool)
ww = np.where(self.solve == 1)[0]
self.mask1[...,ww] = True
self.mask2 = np.zeros_like(self.root.x.state[0]).astype(bool)
ww = np.where(self.solve == 2)[0]
self.mask2[ww] = True
self.nmask1 = self.mask1.sum()
self.nmask2 = self.mask2.sum()
self.wmask2 = np.where(self.mask2)[0]
def sortMask(self):
'''
Sort masks for loading and unloading data
'''
self.root.options.solve = \
sortopt(self.root.options,\
'solve',np.array([1]*len(self.root.x_meta.state)))
self.solve = np.array(self.root.options.solve).copy()
# set up masks for loading and unloading
self.mask1 = np.zeros_like(self.root.x.state).astype(bool)
ww = np.where(self.solve == 1)[0]
self.mask1[..., ww] = True
self.mask2 = np.zeros_like(self.root.x.state[0]).astype(bool)
ww = np.where(self.solve == 2)[0]
self.mask2[ww] = True
self.nmask1 = self.mask1.sum()
self.nmask2 = self.mask2.sum()
self.wmask2 = np.where(self.mask2)[0]
def write_output_file(self, filename, name, info=[]):
'''
Attempt to write state data as ASCII to filename
have a standard interface.
Returns:
is_error
where:
is_error = (error,error_msg)
The file format is flat ASCII with a header
'''
#First check the data is as intended
from eoldas_Lib import sortopt
try:
f = open(filename, 'w')
if self.Data.sd != None:
sd = self.Data.sd.reshape(self.Data.state.shape)
else:
# set to unity
sd = 0. * self.Data.state + 1.0
except IOError:
raise Exception('IOError for %s' % filename)
except:
try:
if 'sd' in self.data.dict():
sd = self.data.sd.reshape(self.data.state.shape)
else:
# set to unity
sd = 0. * self.data.state + 1.0
except:
return (True,'Failed to open file %s for writing with call to %s'%\
(filename,str('write_output_file')))
if 'f' in self.__dict__ and f.errors != None:
error_msg = str(f.errors)
return (True, error_msg)
# make a guess at what the format is
try:
fmt = self.options.result.fmt
except:
try:
fmt = self._state.name.fmt
except:
try:
fmt = self.name.fmt
except:
fmt = None
if fmt == None:
fmt = 'PARAMETER'
try:
state = self.Data.state
except:
state = self.data.state
try:
name = self._state.name
except:
name = self.name
gridded = False
try:
locations = np.array(self.Data.location)
except:
try:
locations = np.array(self.data.location)
except:
# gridded
gridded = True
try:
controls = np.array(self.Data.control)
except:
if not gridded:
controls = np.array(sortopt(self.data, 'control', []))
if gridded and len(np.atleast_1d(state).shape) == 2:
# so no location info
try:
locations = np.arange(state.shape[0])*name.qlocation[0][2]\
+name.qlocation[0][0]
self.data.location = locations
except:
try:
self.Data.location = locations
except:
# need more complex way to sort grid
error_msg = "I can't write this format for this data configuration yet ..."
self.logger.error(error_msg)
return (True, error_msg)
elif gridded:
try:
(locations, qlocations, state, sd) = self.ungrid(state, sd)
except:
raise Exception("You are trying to ungrid a dataset that wasn't gridded using State.regrid()" +\
" so the ungridder information is not available. Either load the data using State.grid " +\
" or set it up some other way or avoid calling this method with this type of data")
n_samples = np.prod(state.shape) / state.shape[-1]
location = np.array(name.location)
control = np.array(name.control)
try:
locations = locations.reshape((n_samples, location.shape[0]))
except:
# really its gridded then ...
# it just thinks it isnt
try:
locations = self.ungridder.location
locations = locations.reshape((n_samples, location.shape[0]))
except:
raise Exception(
'Inconsistency in data shape ... locations is not consistent with state'
)
if len(np.atleast_1d(control)):
try:
controls = controls.reshape((n_samples, control.shape[0]))
except:
if (control == np.array(['mask'])).all():
controls = np.ones(n_samples).reshape(
(n_samples, control.shape[0]))
else:
controls = None
else:
controls = None
bands = np.array(name.state)
nbands = len(np.atleast_1d(bands))
names = np.atleast_1d(np.array(name.state))
grid = state
if fmt == 'BRDF':
'''
The 'old' style UCL BRDF format
A header of :
BRDF n_samples nbands BANDS SD
where:
BANDS wavebands (names or intervals or wavelength, nm)
SD standard deviation (assumed same for all observations)
Data columns:
time days
mask 1 == Good data
VZA degrees
VAA ""
SZA ""
SAA ""
R1
R2 ...
'''
header = "#%s %d %d" % (self.headers[fmt], n_samples, nbands)
for i in xrange(nbands):
header = header + ' ' + str(names[i])
# for i in xrange(nbands):
# header = header + ' ' + str(sd[0,i])
header = header + '\n'
ww = np.where(location == 'time')[0]
if not len(np.atleast_1d(ww)):
location = np.zeros([n_samples, 1])
locations = ['time']
ww = np.where(location == 'time')[0]
timer = locations[:, ww]
try:
ww = np.where(control == 'mask')[0]
except:
ww = []
if not len(np.atleast_1d(ww)):
mask = (0 * timer + 1).astype(int)
elif len(np.atleast_1d(control)):
mask = controls[:, ww].astype(int)
else:
mask = (0 * timer + 1).astype(int)
try:
ww = np.where(control == 'vza')[0]
except:
ww = []
if not len(np.atleast_1d(ww)):
vza = timer * 0
else:
vza = controls[:, ww]
try:
ww = np.where(control == 'sza')[0]
except:
ww = []
if not len(np.atleast_1d(ww)):
sza = timer * 0
else:
sza = controls[:, ww]
try:
ww = np.where(control == 'raa')[0]
except:
ww = []
if not len(np.atleast_1d(ww)):
try:
ww1 = np.where(control == 'saa')[0]
except:
ww1 = []
if not len(np.atleast_1d(ww1)):
saa = timer * 0
else:
saa = controls[:, ww1]
try:
ww1 = np.where(control == 'vaa')[0]
except:
ww1 = []
if not len(np.atleast_1d(ww1)):
vaa = timer * 0
else:
vaa = controls[:, ww1]
else:
vaa = controls[:, ww]
saa = timer * 0
todo = np.array([timer, mask, vza, vaa, sza,
saa]).reshape(6, n_samples).T
todo = np.hstack((todo, grid))
elif fmt == 'BRDF-UCL':
'''
a slighly modified BRDF output format
A header of :
#BRDF-UCL n_samples nbands BANDS SD
where:
BANDS wavebands (names or intervals or wavelength, nm)
SD standard deviation
A 2nd header line of:
#location time row col
or similar n_location location fields.
Data columns:
LOCATION
mask 1 == Good data
VZA degrees
VAA ""
SZA ""
SAA ""
R1
R2 ...
S1
S2 ...
where LOCATION is n_location columns corresponding to
the information in the #location line.
R refers ro reflectance/radiance samples and S to SD
'''
header = "#%s %d %d" % (self.headers[fmt], n_samples, nbands)
header1 = "#%s " % self.headers_2[fmt]
for i in xrange(nbands):
header = header + ' ' + str(names[i])
#for i in xrange(nbands):
# header = header + ' ' + str(sd[0,i])
header = header + '\n'
for i in xrange(len(np.atleast_1d(location))):
header1 = header1 + ' ' + location[i]
header1 = header1 + '\n'
header = header + header1
#ww = np.where(location == 'time')[0]
#if not len(ww):
#location = np.zeros([n_samples,1])
#locations = ['time']
#ww = np.where(location == 'time')[0]
#timer = locations[:,ww]
ww = np.where(control == 'mask')[0]
if not len(np.atleast_1d(ww)):
mask = (0 * timer + 1).astype(int)
else:
mask = controls[:, ww].astype(int)
ww = np.where(control == 'vza')[0]
if not len(np.atleast_1d(ww)):
vza = timer * 0
else:
vza = controls[:, ww]
ww = np.where(control == 'sza')[0]
if not len(np.atleast_1d(ww)):
sza = timer * 0
else:
sza = controls[:, ww]
ww = np.where(control == 'raa')[0]
if not len(np.atleast_1d(ww)):
ww1 = np.where(control == 'saa')[0]
if not len(np.atleast_1d(ww1)):
saa = timer * 0
else:
saa = controls[:, ww1]
ww1 = np.where(control == 'vaa')[0]
if not len(np.atleast_1d(ww1)):
vaa = timer * 0
else:
vaa = controls[:, ww1]
else:
vaa = controls[:, ww]
saa = timer * 0
todo = np.array([mask, vza, vaa, sza, saa]).reshape(5, n_samples).T
todo = np.hstack((locations, todo))
todo = np.hstack((todo, grid))
todo = np.hstack((todo, sd))
elif fmt == 'PARAMETERS':
ndim = len(np.atleast_1d(location))
header = '#%s' % self.headers[fmt]
for i in xrange(ndim):
header = header + ' ' + location[i]
ndim = len(np.atleast_1d(control))
for i in xrange(ndim):
header = header + ' ' + control[i]
for i in xrange(names.size):
header = header + ' ' + names[i]
for i in xrange(names.size):
header = header + ' ' + 'sd-%s' % names[i]
header = header + '\n'
if len(np.atleast_1d(control)):
todo = np.hstack((locations, controls))
else:
todo = locations
grid = grid.reshape(
np.array(grid.shape).prod() / grid.shape[-1], grid.shape[-1])
sd = sd.reshape(
np.array(grid.shape).prod() / grid.shape[-1], grid.shape[-1])
todo = np.hstack((todo, grid))
todo = np.hstack((todo, sd))
elif fmt == 'PARAMETERS-V2':
# dummy
ndim = len(np.atleast_1d(location))
header = '#%s' % self.headers[fmt]
for i in xrange(ndim):
header = header + ' ' + location[i]
for i in xrange(names.size):
header = header + ' ' + names[i]
header = header + '\n'
todo = np.hstack((locations, grid))
else:
f.close()
error_msg = "Unrecognised format for data in write_output_file: %s"\
%fmt
return (True, error_msg)
# bit complex, but done because cant dump header data
# write the header to f (one or two lines)
import tempfile
f.write(header)
# open a tmp file
ff2 = tempfile.NamedTemporaryFile()
# save the data to the tmp file
np.savetxt(ff2.name, todo, fmt=('%11.6f'))
# read in the formatted data as lines & write straight after the header
fa = open(ff2.name)
ff2.close()
f.writelines(fa.readlines())
# close both files ... I dont know if you can close the tmp file earlier
# to ensure flushed output ...?
f.close()
return (False,'Data written to %s with %s'% (filename,str \
('write_output_file')))
def reinit(self,options,names=None,datatype=None,limits=None,\
bounds=None,control=None,location=None,env=None,header=None,\
logdir=None,writers={},grid=False,logger=None,\
datadir=None,logfile=None,name=None,info=[],readers=[],debug=None):
'''
Method to re-initialise the class instance
The setup is on the whole controlled by the datatype which
contains e.g. 'x'. This is used to set up the members
self.x and self.y as SpecialVariables (see SpecialVariable
in eoldas_SpecialVariable.py). There are some special attributes
for datatypes starting with 'y'. These are assumed to be
observational data, which means that when they are read, the
data names associated with them are not limited to those in
self.names but rather set to whatever is read in in the data. This
is because the data names for observational data may be terms such
as waveband names etc that need special interpretation. Also,
the default output format for observational data is
different to that of other data.
The elements self.state is a SpecialVariables which means
that they can be assigned various data types (see SpecialVariables)
and loaded accordingly (e.g. if a filename is specified, this is
read in to the data structure. The SpecialVariables contain 'hidden'
datasets, which here are mainly the 'control' and 'location'
information. A SpecialVariable has two internal structures: `data`
and `name`. The former is used to store data values (e.g. the
state values) and the latter to store associated metadata.
For example, `control` is passed here e.g. as [`mask`,`vza`] and this
gives the metadata that are stored in `name`. The actual values
of the control data are stored in the `data` section. For
location, we might be passed [`time`,`row`,`col`], so this is set
in names.location, and the data.location contains the values
of the location at each of these elements. For the actual state
dataset, this is stored according to its name, so for `x` the values
are stored in data.x and the associated data names in name.x.
State datasets must represent at least the mean and standard deviation
of a state for them to be of value in EOLDAS. TThe mean is accessed
as e.g. self.state for the state dataset.
The sd is accessed can be accessed as self._state.sd if it
has been set. This reference can also be used to directly set
data associated with a SpecialVariable, e.g.
self.Data.control = np.zeros([2,3])
to represent 2 samples with 3 control variables. You can access name
information similarly with
print self.Name.control
but this will generate a KeyError if the term has not been set. You
can check it exists with:
key = 'control'
if key in self.Name:
this = (self.Data[key],self.Name[key])
To get e.g. a dictionary representation of a SpecialVariable
you can use eg:
self.Name.to_dict()
to get the name dictionary, or
thisdict = self._state.to_dict()
to get the full representation, which then contains 'data' and
'name' as well as some other information stored in the
SpecialVariable.
You can similarly load them using e.g.
self.Data.update(
ParamStorage().from_dict(thisdict['data'])
combine=True)
'''
# set up a fakes dictionary from the data types
self.set('datatype',datatype)
self.set('fakes', {'state':'_state'})
# first check that options is sensible
self.__check_type(options,ParamStorage,fatal=True)
self.options = options
from eoldas_Lib import set_default_limits,\
check_limits_valid,quantize_location, sortopt
nSpecial = 1
if name == None:
import time
thistime = str(time.time())
name = type(self).__name__
name = "%s.%s" % (name,thistime)
self.thisname = name
self.options.thisname = str(name).replace(' ','_')
log_terms = {\
'logfile':logfile or sortopt(self.options,'logfile',None),\
'logdir':logdir or sortopt(self.options,'logdir',None),\
'debug' : debug or sortopt(self.options,'debug',True)}
self.datadir = datadir or sortopt(self.options,'datadir',["."])
self.header = header or "EOLDAS pickle V1.0 - plewis"
env = env or sortopt(self.options,'env',None)
names = names or sortopt(self.options,'names',None)
location = location or sortopt(self.options,'location',['time'])
control = control or sortopt(self.options,'control',[])
limits = limits or sortopt(self.options,'limits',\
set_default_limits(np.array(location)))
limits = limits or self.options.limits
limits = np.array(check_limits_valid(limits))
bounds = bounds or sortopt(self.options,'bounds',\
[[None,None]] * xlen(names))
self.options.bounds = bounds
self.headers = {'PARAMETERS-V2':"PARAMETERS-V2", \
'PARAMETERS':"PARAMETERS", \
'BRDF-UCL':'BRDF-UCL',\
'BRDF': 'BRDF'}
self.headers_2 = {'BRDF-UCL':'location'}
# The ones pre-loaded are
# self.read_functions = [self.read_pickle,self.read_numpy_fromfile]
self._state = SpecialVariable(info=info,name=self.thisname,\
readers=readers,datadir=self.datadir,\
env=env,writers=writers,\
header=self.header,\
logger=logger,log_terms=log_terms,\
simple=False)
# self._state is where data are read into
# but self.Data and self.Name are where we access them from
self.grid=grid
# this is so we can access this object from
# inside a SpecialVariable
self.state = np.array([0.])
# a default data fmt output
if datatype[0] == 'y':
self.Name.fmt = 'BRDF'
self.Name.state = np.array(['dummy'])
else:
self.Name.fmt = 'PARAMETERS'
n_params = xlen(names)
if not n_params:
error_msg = \
"The field 'names' must be defined in options or"+ \
"passed directly to this method if you have the data type x"
raise Exception(error_msg)
self.Name.state = np.array(names)
self.Name.location = np.array(location)
self.Name.control = np.array(control)
self.Name.header = self.header
self.Name.bounds = np.array(bounds)
self.Name.qlocation = np.array(limits)
self.Name.datadir = datadir
#
# sort this object's name
# sort logging
self.logger = sortlog(self,log_terms['logfile'],logger,name=self.thisname,
logdir=log_terms['logdir'],debug=log_terms['debug'])
self.logger.info('Initialising %s' % type(self).__name__)
def write_output_file(self,filename,name,info=[]):
'''
Attempt to write state data as ASCII to filename
have a standard interface.
Returns:
is_error
where:
is_error = (error,error_msg)
The file format is flat ASCII with a header
'''
#First check the data is as intended
from eoldas_Lib import sortopt
try:
f = open(filename,'w')
if self.Data.sd != None:
sd = self.Data.sd.reshape(self.Data.state.shape)
else:
# set to unity
sd = 0.*self.Data.state + 1.0
except IOError:
raise Exception('IOError for %s'%filename)
except:
try:
if 'sd' in self.data.dict():
sd = self.data.sd.reshape(self.data.state.shape)
else:
# set to unity
sd = 0.*self.data.state + 1.0
except:
return (True,'Failed to open file %s for writing with call to %s'%\
(filename,str('write_output_file')))
if 'f' in self.__dict__ and f.errors != None:
error_msg = str(f.errors)
return (True,error_msg)
# make a guess at what the format is
try:
fmt = self.options.result.fmt
except:
try:
fmt = self._state.name.fmt
except:
try:
fmt = self.name.fmt
except:
fmt = None
if fmt == None:
fmt = 'PARAMETER'
try:
state = self.Data.state
except:
state = self.data.state
try:
name = self._state.name
except:
name = self.name
gridded = False
try:
locations = np.array(self.Data.location)
except:
try:
locations = np.array(self.data.location)
except:
# gridded
gridded = True
try:
controls = np.array(self.Data.control)
except:
if not gridded:
controls = np.array(sortopt(self.data,'control',[]))
if gridded and len(np.atleast_1d(state).shape) == 2:
# so no location info
try:
locations = np.arange(state.shape[0])*name.qlocation[0][2]\
+name.qlocation[0][0]
self.data.location = locations
except:
try:
self.Data.location = locations
except:
# need more complex way to sort grid
error_msg = "I can't write this format for this data configuration yet ..."
self.logger.error(error_msg)
return (True,error_msg)
elif gridded:
try:
(locations,qlocations,state,sd) = self.ungrid(state,sd)
except:
raise Exception("You are trying to ungrid a dataset that wasn't gridded using State.regrid()" +\
" so the ungridder information is not available. Either load the data using State.grid " +\
" or set it up some other way or avoid calling this method with this type of data")
n_samples = np.prod(state.shape)/state.shape[-1]
location = np.array(name.location)
control = np.array(name.control)
try:
locations = locations.reshape((n_samples,location.shape[0]))
except:
# really its gridded then ...
# it just thinks it isnt
try:
locations = self.ungridder.location
locations = locations.reshape((n_samples,location.shape[0]))
except:
raise Exception('Inconsistency in data shape ... locations is not consistent with state')
if len(np.atleast_1d(control)):
try:
controls = controls.reshape((n_samples,control.shape[0]))
except:
if (control == np.array(['mask'])).all():
controls = np.ones(n_samples).reshape((n_samples,control.shape[0]))
else:
controls = None
else:
controls = None
bands = np.array(name.state)
nbands = len(np.atleast_1d(bands))
names = np.atleast_1d(np.array(name.state))
grid = state
if fmt == 'BRDF':
'''
The 'old' style UCL BRDF format
A header of :
BRDF n_samples nbands BANDS SD
where:
BANDS wavebands (names or intervals or wavelength, nm)
SD standard deviation (assumed same for all observations)
Data columns:
time days
mask 1 == Good data
VZA degrees
VAA ""
SZA ""
SAA ""
R1
R2 ...
'''
header = "#%s %d %d"%(self.headers[fmt],n_samples,nbands)
for i in xrange(nbands):
header = header + ' ' + str(names[i])
# for i in xrange(nbands):
# header = header + ' ' + str(sd[0,i])
header = header + '\n'
ww = np.where(location == 'time')[0]
if not len(np.atleast_1d(ww)):
location = np.zeros([n_samples,1])
locations = ['time']
ww = np.where(location == 'time')[0]
timer = locations[:,ww]
try:
ww = np.where(control == 'mask')[0]
except:
ww = []
if not len(np.atleast_1d(ww)):
mask = (0*timer+1).astype(int)
elif len(np.atleast_1d(control)):
mask = controls[:,ww].astype(int)
else:
mask = (0*timer+1).astype(int)
try:
ww = np.where(control== 'vza')[0]
except:
ww = []
if not len(np.atleast_1d(ww)):
vza = timer*0
else:
vza = controls[:,ww]
try:
ww = np.where(control== 'sza')[0]
except:
ww = []
if not len(np.atleast_1d(ww)):
sza = timer*0
else:
sza = controls[:,ww]
try:
ww = np.where(control== 'raa')[0]
except:
ww = []
if not len(np.atleast_1d(ww)):
try:
ww1 = np.where(control== 'saa')[0]
except:
ww1 = []
if not len(np.atleast_1d(ww1)):
saa = timer*0
else:
saa = controls[:,ww1]
try:
ww1 = np.where(control== 'vaa')[0]
except:
ww1 = []
if not len(np.atleast_1d(ww1)):
vaa = timer*0
else:
vaa = controls[:,ww1]
else:
vaa = controls[:,ww]
saa = timer*0
todo = np.array([timer,mask,vza,vaa,sza,saa]).reshape(6,n_samples).T
todo = np.hstack((todo,grid))
elif fmt == 'BRDF-UCL':
'''
a slighly modified BRDF output format
A header of :
#BRDF-UCL n_samples nbands BANDS SD
where:
BANDS wavebands (names or intervals or wavelength, nm)
SD standard deviation
A 2nd header line of:
#location time row col
or similar n_location location fields.
Data columns:
LOCATION
mask 1 == Good data
VZA degrees
VAA ""
SZA ""
SAA ""
R1
R2 ...
S1
S2 ...
where LOCATION is n_location columns corresponding to
the information in the #location line.
R refers ro reflectance/radiance samples and S to SD
'''
header = "#%s %d %d"%(self.headers[fmt],n_samples,nbands)
header1 = "#%s "%self.headers_2[fmt]
for i in xrange(nbands):
header = header + ' ' + str(names[i])
#for i in xrange(nbands):
# header = header + ' ' + str(sd[0,i])
header = header + '\n'
for i in xrange(len(np.atleast_1d(location))):
header1 = header1 + ' ' + location[i]
header1 = header1 + '\n'
header = header + header1
#ww = np.where(location == 'time')[0]
#if not len(ww):
#location = np.zeros([n_samples,1])
#locations = ['time']
#ww = np.where(location == 'time')[0]
#timer = locations[:,ww]
ww = np.where(control == 'mask')[0]
if not len(np.atleast_1d(ww)):
mask = (0*timer+1).astype(int)
else:
mask = controls[:,ww].astype(int)
ww = np.where(control== 'vza')[0]
if not len(np.atleast_1d(ww)):
vza = timer*0
else:
vza = controls[:,ww]
ww = np.where(control== 'sza')[0]
if not len(np.atleast_1d(ww)):
sza = timer*0
else:
sza = controls[:,ww]
ww = np.where(control== 'raa')[0]
if not len(np.atleast_1d(ww)):
ww1 = np.where(control== 'saa')[0]
if not len(np.atleast_1d(ww1)):
saa = timer*0
else:
saa = controls[:,ww1]
ww1 = np.where(control== 'vaa')[0]
if not len(np.atleast_1d(ww1)):
vaa = timer*0
else:
vaa = controls[:,ww1]
else:
vaa = controls[:,ww]
saa = timer*0
todo = np.array([mask,vza,vaa,sza,saa]).reshape(5,n_samples).T
todo = np.hstack((locations,todo))
todo = np.hstack((todo,grid))
todo = np.hstack((todo,sd))
elif fmt == 'PARAMETERS':
ndim = len(np.atleast_1d(location))
header = '#%s'%self.headers[fmt]
for i in xrange(ndim):
header = header + ' ' + location[i]
ndim = len(np.atleast_1d(control))
for i in xrange(ndim):
header = header + ' ' + control[i]
for i in xrange(names.size):
header = header + ' ' + names[i]
for i in xrange(names.size):
header = header + ' ' + 'sd-%s'%names[i]
header = header + '\n'
if len(np.atleast_1d(control)):
todo = np.hstack((locations,controls))
else:
todo = locations
grid = grid.reshape(np.array(grid.shape).prod()/grid.shape[-1],grid.shape[-1])
sd = sd.reshape(np.array(grid.shape).prod()/grid.shape[-1],grid.shape[-1])
todo = np.hstack((todo,grid))
todo = np.hstack((todo,sd))
elif fmt == 'PARAMETERS-V2':
# dummy
ndim = len(np.atleast_1d(location))
header = '#%s'%self.headers[fmt]
for i in xrange(ndim):
header = header + ' ' + location[i]
for i in xrange(names.size):
header = header + ' ' + names[i]
header = header + '\n'
todo = np.hstack((locations,grid))
else:
f.close()
error_msg = "Unrecognised format for data in write_output_file: %s"\
%fmt
return (True,error_msg)
# bit complex, but done because cant dump header data
# write the header to f (one or two lines)
import tempfile
f.write(header)
# open a tmp file
ff2 = tempfile.NamedTemporaryFile()
# save the data to the tmp file
np.savetxt(ff2.name,todo,fmt=('%11.6f'))
# read in the formatted data as lines & write straight after the header
fa = open(ff2.name)
ff2.close()
f.writelines(fa.readlines())
# close both files ... I dont know if you can close the tmp file earlier
# to ensure flushed output ...?
f.close()
return (False,'Data written to %s with %s'% (filename,str \
('write_output_file')))
def prep(self,thisconf):
'''
A method to prepare the solver
'''
self.root = self.confs.root[thisconf]
root = self.confs.root[thisconf]
self.sortMask()
self.op = sortopt(root.general,'optimisation',ParamStorage())
self.op.plot = sortopt(self.op,'plot',0)
self.op.name = sortopt(self.op,'name','solver')
self.op.maxfunevals = sortopt(self.op,'maxfunevals',2e4)
self.op.maxiter = sortopt(self.op,'maxiter',1e4)
self.op.gtol = sortopt(self.op,'gtol',1e-3)
self.op.iprint = sortopt(self.op,'iprint',1)
self.op.solverfn = sortopt(self.op,'solverfn','scipy_lbfgsb')
self.op.randomise = sortopt(self.op,'randomise',False)
self.op.no_df = sortopt(self.op,'no_df',False)
self.result = sortopt(root.options,'result',ParamStorage())
self.result.filename = sortopt(root.options.result,'filename',\
'results.pkl')
self.result.fmt = sortopt(root.options.result,'format','pickle')
try:
self.transform = self.root.options.x.transform
self.invtransform = self.root.options.x.transform
except:
self.transform = None
self.invtransform = None
# descend into the operators and identify any observation ops
# we then want to be able to write out files (or plot data)
# that are H(x).
# We only do this for Operators that have both 'x' and 'y'
# terms as we store the filename under 'y.result'
self.Hx_ops = []
for i,op in enumerate(root.operators):
op.loader(root)
if 'y_meta' in op.dict() and op.options.y.datatype == 'y':
# There is a potential observation
op.y_state.options.y.result = \
sortopt(op.y_state.options.y,'result',ParamStorage())
op.y_state.options.y.result.filename = \
sortopt(op.y_state.options.y.result,\
'filename',op.y_state.thisname)
op.y_state.options.y.result.format = \
sortopt(op.y_state.options.y.result,\
'format','PARAMETERS')
state = op.y_state._state
this = { \
'filename':op.y_state.options.y.result.filename,\
'format':op.y_state.options.y.result.format,\
'state':state,\
'y':op.y_state,\
'op':op,\
'transform':self.transform,\
'invtransform':self.invtransform,\
'Hx':op.linear.H}
op.Hx_info = this
self.Hx_ops.append(this)
else:
this = { \
'transform':self.transform,\
'invtransform':self.invtransform,\
}
op.Hx_info = this
def __init__(self,options,name=None):
'''
The class initialiser
This sets up the problem, i.e. loads the spectral information from
an options structure.
The options structure will contain:
y_meta.names:
--------------------
The names of the state vector elements for a y State as string list.
These are interpreted as wavebands associated with the y state.
They may be:
1. the names of bandpass functions e.g. MODIS-b2
2. Wavelength intervals e.g. "450-550:
3. Single wavelength e.g. 451
In the case of 2 or 3, we interpret the wavelengths
directly from the string. For defined bandpass functions
we look in a library to interpret the functions.
Optional:
options.operator_spectral
--------------------
(lmin,lmax,lstep) for the operator. If this is *not* specified
then the model is assumed to be 'non spectral' and a flag
self.is_spectral is set False. In this case, we form a new set
of state variables to solve for and don't bother setting up
all of the bandpass information. This means that we have to inform
the parameters operator to set up a new set of state vectors.
These are based on the bandpass names self.bandnames and
the names in options.x_meta.names.
options.datadir
--------------------
Data directory for bandpass files
options.bandpass_libraries
--------------------
A list of filenames containing band pass libraries
options.bandpass_library
--------------------
A dictionary of existing bandpass functions
This makes calls to:
self.setup_spectral_config()
self.load_bandpass_library()
self.load_bandpass_from_pulses()
self.normalise_wavebands()
'''
from eoldas_Lib import isfloat,sortopt
if name == None:
import time
thistime = str(time.time())
name = type(self).__name__
name = "%s.%s" % (name,thistime)
self.thisname = name
self.options = options
self.logger = logging.getLogger (self.thisname+'.spectral')
# set self.nlw, self.bandnames & associated size terms
# we need this to discretise the bandpass functions
self.setup_spectral_config(self.options.y_meta.state)
self.is_spectral=sortopt(options.general,'is_spectral',True)
if not self.is_spectral:
self.logger.info("Non spectral operator specified")
return
self.logger.info("Spectral operator specified")
if 'datadir' in self.options.dict():
self.datadir = self.options.datadir
else:
self.datadir = ['.','~/.eoldas','..']
self.logger.info("Data directories: %s"%str(self.datadir))
if 'bandpass_library' in self.options.dict():
self.logger.info("Receiving existing bandpass library")
self.bandpass_library = self.options.bandpass_library
else:
self.logger.info("Starting new bandpass library")
self.bandpass_library={}
if 'bandpass_libraries' in self.options.dict():
self.bandpass_libraries = self.options.bandpass_libraries
for i in self.bandpass_libraries:
self.logger.info("Attempting to load bandpass library %s"%i)
self.bandpass_library,is_error = \
self.load_bandpass_library(i,self.bandpass_library)
if is_error[0]:
self.logger.error\
('Cannot load file %s loaded into bandpass library'%i)
self.logger.error(is_error[1])
else:
self.logger.info("No bandpass libraries specified")
self.lstep = sortopt(self,'lstep',1)
self.nlw = sortopt(self,'nlw',None)
# if you havent defined the wavelength range, use 0-10000
if self.nlw == None:
self.logger.info("No wavelength bounds defined")
self.logger.info("setting as [0,10000,1]")
self.nlw = np.arange(10000)
self.pseudowavelengths = []
# Now we can start to load the band names
for i in self.bandnames:
# check to see if its in the library:
if not i in self.bandpass_library:
# try to interpret it
ok,this = isfloat(i)
if ok:
self.bandpass_library, bandpass_name = \
self.load_bandpass_from_pulses(str(i),\
this,0.5*self.lstep,self.nlw,\
self.bandpass_library,\
self.lstep)
else:
# Try splitting it
that = i.split('-')
if len(that) == 2 and isfloat(that[0])[0] \
and isfloat(that[1])[0]:
f0 = float(that[0])
f1 = float(that[1])
self.bandpass_library, bandpass_name = \
self.load_bandpass_from_pulses(i,\
0.5*(f1+f0),0.5*(f1-f0),self.nlw,\
self.bandpass_library,\
self.lstep)
else:
f0 = len(self.pseudowavelengths) + self.nlw[0]
bandpass_name = i
self.logger.info("******************************************************")
self.logger.info("requested waveband %s is not in the spectral library"%i)
self.logger.info("and cannot be interpreted as a wavelength or wavelength range")
self.logger.info("so we will set it up as a pseudowavelength and hope that")
self.logger.info("any operator you use does not need to interpret its value")
self.logger.info("for reference:")
self.logger.info(" band %s"%i)
self.logger.info("is interpreted here as:")
self.bandpass_library, dummy = \
self.load_bandpass_from_pulses(str(f0),\
f0,self.lstep*0.5,self.nlw,\
self.bandpass_library,\
self.lstep)
self.logger.info(" %d nm"%f0)
self.logger.info("******************************************************")
self.pseudowavelengths.append(i)
try:
self.bandpass_library[i] = self.bandpass_library[dummy]
except:
self.bandpass_library[i] = self.bandpass_library[str(f0)]
# Now post-process the bandpass library
self.normalise_wavebands(self.bandnames)
def _load_rt_library(self, rt_library):
"""
A method that loads up the compiled RT library code and sets some
configuration options. This method tries to import all the methods and
make them available through `self.rt_library.<method_name>`. It is also
a safe importer: if some functions are not available in the library,
it will provide safe methods from them. Additionally, while importing,
a number of configuration options in the class are also updated or set
to default values.
Parameters
-----------
rt_library : string
This is the name of the library object (.so file) that will be
loaded
"""
from eoldas_Lib import sortopt
import_string = "from %s import " % (rt_library)
self.logger.debug("Using %s..." % rt_library)
self.rt_library = sortopt(self, 'rt_library', ParamStorage())
# 1. Import main functionality
try:
self.logger.debug("Loading rt_model")
exec(import_string + "rt_model")
self.rt_library.rt_model = rt_model
except ImportError:
self.logger.info(\
"Could not import basic RT functionality: rt_model")
self.logger.info(\
"Check library paths, and whether %s.so is available" % \
rt_library)
raise Exception('error importing library %s' % rt_library)
# 1a. Import conditioning methods that can be ignored
try:
exec(import_string + 'rt_modelpre')
self.rt_library.rt_modelpre = rt_modelpre
except:
self.rt_library.rt_modelpre = self._nowt
try:
exec(import_string + 'rt_modelpre')
self.rt_library.rt_modelpost = rt_modelpre
except:
self.rt_library.rt_modelpost = self._nowt
# 2. Try to import derivative
self.rt_model.ignore_derivative = sortopt(self.rt_model,\
'ignore_derivative',False)
if self.rt_model.ignore_derivative == False:
try:
exec ( import_string + \
"rt_modeld, rt_modeldpre, rt_modeldpost" + \
", rt_modelpred" )
self.rt_model.have_adjoint = True
self.J_prime = self.J_prime_full
self.rt_library.rt_modeld = rt_modeld
self.rt_library.rt_modeldpre = rt_modeldpre
self.rt_library.rt_modeldpost = rt_modeldpost
self.rt_library.rt_modelpred = rt_modelpred
except ImportError:
self.logger.info(
"No adjoint. Using finite differences approximation.")
self.rt_model.have_adjoint = False
else:
self.logger.info(
"ignoring adjoint. Using finite differences approximation.")
self.rt_model.have_adjoint = False
self._configure_adjoint()
try:
exec(import_string + "rt_model_deriv")
self.rt_library.rt_model_deriv = rt_model_deriv
except ImportError:
self.rt_library.rt_model_deriv = None
def prep(self, thisconf):
'''
A method to prepare the solver
'''
self.root = self.confs.root[thisconf]
root = self.confs.root[thisconf]
self.sortMask()
self.op = sortopt(root.general, 'optimisation', ParamStorage())
self.op.plot = sortopt(self.op, 'plot', 0)
self.op.name = sortopt(self.op, 'name', 'solver')
self.op.maxfunevals = sortopt(self.op, 'maxfunevals', 2e4)
self.op.maxiter = sortopt(self.op, 'maxiter', 1e4)
self.op.gtol = sortopt(self.op, 'gtol', 1e-3)
self.op.iprint = sortopt(self.op, 'iprint', 1)
self.op.solverfn = sortopt(self.op, 'solverfn', 'scipy_lbfgsb')
self.op.randomise = sortopt(self.op, 'randomise', False)
self.op.no_df = sortopt(self.op, 'no_df', False)
self.result = sortopt(root.options, 'result', ParamStorage())
self.result.filename = sortopt(root.options.result,'filename',\
'results.pkl')
self.result.fmt = sortopt(root.options.result, 'format', 'pickle')
try:
self.transform = self.root.options.x.transform
self.invtransform = self.root.options.x.transform
except:
self.transform = None
self.invtransform = None
# descend into the operators and identify any observation ops
# we then want to be able to write out files (or plot data)
# that are H(x).
# We only do this for Operators that have both 'x' and 'y'
# terms as we store the filename under 'y.result'
self.Hx_ops = []
for i, op in enumerate(root.operators):
op.loader(root)
if 'y_meta' in op.dict() and op.options.y.datatype == 'y':
# There is a potential observation
op.y_state.options.y.result = \
sortopt(op.y_state.options.y,'result',ParamStorage())
op.y_state.options.y.result.filename = \
sortopt(op.y_state.options.y.result,\
'filename',op.y_state.thisname)
op.y_state.options.y.result.format = \
sortopt(op.y_state.options.y.result,\
'format','PARAMETERS')
state = op.y_state._state
this = { \
'filename':op.y_state.options.y.result.filename,\
'format':op.y_state.options.y.result.format,\
'state':state,\
'y':op.y_state,\
'op':op,\
'transform':self.transform,\
'invtransform':self.invtransform,\
'Hx':op.linear.H}
op.Hx_info = this
self.Hx_ops.append(this)
else:
this = { \
'transform':self.transform,\
'invtransform':self.invtransform,\
}
op.Hx_info = this
def __init__(self,
confs,
logger=None,
logfile=None,
thisname=None,
name=None,
datadir=None,
logdir=None):
'''
Initialise the solver.
This does the following:
1. Read configuration file(s)
2. Load operators
3. Test the call to the cost function
There can be multiple groups of configuration files, so
self.confs, that holds the core information setup here
can contain multiple configurations.
The number of configurations is len(confs.infos) and
the ith configuration is conf = self.confs.infos[i].
Various loggers are available throughout the classes
used, but the top level logger is self.confs.logger,
so you can log with e.g.
self.confs.logger.info('this is some info')
The root operator is stored in self.confs.root[i]
for the ith configuration, so the basic call to the cost
function is:
J,J_prime = self.confs[i].parameter.cost(None)
'''
from eoldas_ConfFile import ConfFile
from eoldas_Lib import sortopt
name = name or thisname
if name == None:
import time
thistime = str(time.time())
name = type(self).__name__
name = "%s.%s" % (name, thistime)
self.thisname = name
self.confs = confs
self.top = sortopt(self, 'top', ParamStorage())
self.top.general = sortopt(self.top, 'general', ParamStorage())
thisname = sortopt(self.top.general, 'name', thisname or self.thisname)
logfile = sortopt(self.top.general, 'logfile', logfile or 'log.dat')
logdir = sortopt(self.top.general, 'logdir', logfile or 'logs')
datadir = sortopt(self.top.general, 'datadir', datadir or ['.'])
self.logger = sortlog(self,logfile,logger or self.confs.logger,\
name=self.thisname,logdir=logdir,debug=True)
n_configs = len(self.confs.infos)
self.confs.root = []
self.have_unc = False
try:
logdir = logdir
except:
logdir = self.top.general.logdir
# first set up parameter
conf = confs.infos[0]
general = conf.general
op = conf.parameter
if not 'parameter' in conf.dict():
raise Exception('No parameter field found in %s item %d'%\
(conf.__doc__,0))
general.is_spectral = sortopt(general, 'is_spectral', True)
if not general.is_spectral:
sort_non_spectral_model(op, conf.operator, logger=confs.logger)
general.init_test = sortopt(general, 'init_test', False)
confs.logger.info('loading parameter state')
conf.parameter.name = 'Operator'
parameter = eval(op.name)(op,general,\
parameter=None,\
logger=confs.logger,\
name=name+".parameter",\
datatype=list(conf.parameter.datatypes),\
logdir=logdir,\
logfile=logfile,\
datadir=datadir)
try:
parameter.transform = parameter.options.x.transform
parameter.invtransform = parameter.options.x.invtransform
except:
parameter.transform = parameter.options.x.names
parameter.invtransform = parameter.options.x.names
# we now have access to parameter.x.state, parameter.x.sd etc
# and possibly parameter.y.state etc.
operators = []
for (opname, op) in conf.operator.dict().iteritems():
if opname != 'helper':
#pdb.set_trace()
exec('from eoldas_%s import %s' % (op.name, op.name))
# make sure the data limits and x bounds are the same
# ... inherit from parameter
op.limits = parameter.options.limits
if not 'datatypes' in op.dict():
op.datatypes = 'x'
thisop = eval(op.name)(op,general,parameter=parameter,\
logger=confs.logger,\
name=name+".%s-%s"%(thisname,opname),\
datatype=list(op.datatypes),\
logdir=logdir,\
logfile=logfile,\
datadir=datadir)
# load from parameter
thisop.loader(parameter)
operators.append(thisop)
try:
thisop.transform = parameter.options.x.transform
thisop.invtransform = parameter.options.x.invtransform
except:
thisop.transform = parameter.options.x.names
thisop.invtransform = parameter.options.x.names
thisop.ploaderMask = np.in1d(parameter.x_meta.state,
thisop.x_meta.state)
try:
thisop.invtransform = np.array(
thisop.invtransform[thisop.ploaderMask])
thisop.transform = np.array(
thisop.transform[thisop.ploaderMask])
except:
ww = thisop.ploaderMask
thisop.invtransform = np.array(thisop.transform)[ww]
thisop.transform = np.array(thisop.transform)[ww]
# sort the loaders
parameter.operators = operators
self.confs.root.append(parameter)
# Now we have set up the operators
# try out the cost function
if general.init_test:
self.logger.info('testing cost function calls')
J, J_prime = self.confs.root[0].cost()
self.logger.info('done')
self.confs.root = np.array(self.confs.root)
def postload_prepare(self):
'''
This is called on initialisation, after data have been read in
Here, we load parameters specifically associated with the
model H(x).
In the case of this differential operator, there are:
model_order : order of the differential operator
(integer)
wraparound : edge conditions
Can be:
periodic
none
reflexive
lag : The (time/space) lag at which
the finite difference is calculated
in the differential operator here.
If this is 1, then we take the difference
between each sample point and its neighbour.
This is what we normally use. The main
purpose of this mechanism is to allow
differences at multiple lags to be
calculated (fuller autocorrelation function
constraints as in kriging)
Multiple lags can be specified (which you could
use to perform kriging), in which case lag
weight should also be specified.
lag_weight : The weight associated with each lag. This will
generally be decreasing with increasing lag for
a 'usual' autocorrelation function. There is no point
specifying this if only a single lag is specified
as the function is normalised.
If the conditions are specified as periodic
the period of the function can also be specified, e.g.
for time varying data, you could specify 365 for the
periodic period.
These are specified in the configuration file as
operator.modelt.rt_model.model_order
operator.modelt.rt_model.wraparound
operator.modelt.rt_model.lag
operator.modelt.rt_model.lag_weight
The default values (set here) are 1, 'none', 1 and 1 respectively.
To specify the period for `periodic` specify e.g.:
[operator.modelt.rt_model]
wraparound=periodic,365
The default period is set to 0, which implies that it is periodic
on whatever the data extent is.
Or for multiple lags:
[operator.modelt.rt_model]
lag=1,2,3,4,5
lag_weight=1,0.7,0.5,0.35,0.2
NB this lag mechanism has not yet been fully tested
and should be used with caution. It is intended more
as a placeholder for future developments.
Finally, we can also decide to work with
inverse gamma (i.e. an uncertainty-based measure)
This is achieved by setting the flag
operator.modelt.rt_model.inverse_gamma=True
This flag should be set if you intend to estimate gamma in the
Data Assimilation. Again, the is experimental and should be used
with caution.
'''
from eoldas_Lib import sortopt
self.rt_model = sortopt(self.options, 'rt_model', ParamStorage())
self.rt_model.lag = sortopt(self.rt_model, 'lag', 1)
self.rt_model.inverse_gamma= \
sortopt(self.rt_model,'inverse_gamma',False)
self.rt_model.model_order = \
sortopt(self.rt_model,'model_order',1)
self.rt_model.wraparound = \
sortopt(self.rt_model,'wraparound','none')
self.rt_model.wraparound_mod = 0
if np.array(self.rt_model.wraparound).size == 2 and \
np.array(self.rt_model.wraparound)[0] == 'periodic':
self.rt_model.wraparound_mod = \
np.array(self.rt_model.wraparound)[1]
self.rt_model.wraparound = \
np.array(self.rt_model.wraparound)[0]
self.rt_model.lag = \
sortopt(self.rt_model,'lag',[1])
self.rt_model.lag = np.array(self.rt_model.lag).flatten()
self.rt_model.lag_weight = \
sortopt(self.rt_model,'lag_weight',[1.]*\
self.rt_model.lag.size)
self.rt_model.lag_weight = np.array(\
self.rt_model.lag_weight).flatten().astype(float)
if self.rt_model.lag_weight.sum() == 0:
self.rt_model.lag_weight[:] = np.ones(
self.rt_model.lag_weight.size)
self.rt_model.lag_weight = self.rt_model.lag_weight\
/ self.rt_model.lag_weight.sum()
def postload_prepare(self):
'''
This is called on initialisation, after data have been read in
Here, we load parameters specifically associated with the
model H(x).
In the case of this differential operator, there are:
model_order : order of the differential operator
(integer)
wraparound : edge conditions
Can be:
periodic
none
reflexive
lag : The (time/space) lag at which
the finite difference is calculated
in the differential operator here.
If this is 1, then we take the difference
between each sample point and its neighbour.
This is what we normally use. The main
purpose of this mechanism is to allow
differences at multiple lags to be
calculated (fuller autocorrelation function
constraints as in kriging)
Multiple lags can be specified (which you could
use to perform kriging), in which case lag
weight should also be specified.
lag_weight : The weight associated with each lag. This will
generally be decreasing with increasing lag for
a 'usual' autocorrelation function. There is no point
specifying this if only a single lag is specified
as the function is normalised.
If the conditions are specified as periodic
the period of the function can also be specified, e.g.
for time varying data, you could specify 365 for the
periodic period.
These are specified in the configuration file as
operator.modelt.rt_model.model_order
operator.modelt.rt_model.wraparound
operator.modelt.rt_model.lag
operator.modelt.rt_model.lag_weight
The default values (set here) are 1, 'none', 1 and 1 respectively.
To specify the period for `periodic` specify e.g.:
[operator.modelt.rt_model]
wraparound=periodic,365
The default period is set to 0, which implies that it is periodic
on whatever the data extent is.
Or for multiple lags:
[operator.modelt.rt_model]
lag=1,2,3,4,5
lag_weight=1,0.7,0.5,0.35,0.2
NB this lag mechanism has not yet been fully tested
and should be used with caution. It is intended more
as a placeholder for future developments.
Finally, we can also decide to work with
inverse gamma (i.e. an uncertainty-based measure)
This is achieved by setting the flag
operator.modelt.rt_model.inverse_gamma=True
This flag should be set if you intend to estimate gamma in the
Data Assimilation. Again, the is experimental and should be used
with caution.
'''
from eoldas_Lib import sortopt
self.rt_model = sortopt(self.options,'rt_model',ParamStorage())
self.rt_model.lag = sortopt(self.rt_model,'lag',1)
self.rt_model.inverse_gamma= \
sortopt(self.rt_model,'inverse_gamma',False)
self.rt_model.model_order = \
sortopt(self.rt_model,'model_order',1)
self.rt_model.wraparound = \
sortopt(self.rt_model,'wraparound','none')
self.rt_model.wraparound_mod = 0
if np.array(self.rt_model.wraparound).size == 2 and \
np.array(self.rt_model.wraparound)[0] == 'periodic':
self.rt_model.wraparound_mod = \
np.array(self.rt_model.wraparound)[1]
self.rt_model.wraparound = \
np.array(self.rt_model.wraparound)[0]
self.rt_model.lag = \
sortopt(self.rt_model,'lag',[1])
self.rt_model.lag = np.array(self.rt_model.lag).flatten()
self.rt_model.lag_weight = \
sortopt(self.rt_model,'lag_weight',[1.]*\
self.rt_model.lag.size)
self.rt_model.lag_weight = np.array(\
self.rt_model.lag_weight).flatten().astype(float)
if self.rt_model.lag_weight.sum() == 0:
self.rt_model.lag_weight[:] = np.ones(self.rt_model.lag_weight.size)
self.rt_model.lag_weight = self.rt_model.lag_weight\
/ self.rt_model.lag_weight.sum()
def __init__(self,confs,logger=None,logfile=None,thisname=None,name=None,datadir=None,logdir=None):
'''
Initialise the solver.
This does the following:
1. Read configuration file(s)
2. Load operators
3. Test the call to the cost function
There can be multiple groups of configuration files, so
self.confs, that holds the core information setup here
can contain multiple configurations.
The number of configurations is len(confs.infos) and
the ith configuration is conf = self.confs.infos[i].
Various loggers are available throughout the classes
used, but the top level logger is self.confs.logger,
so you can log with e.g.
self.confs.logger.info('this is some info')
The root operator is stored in self.confs.root[i]
for the ith configuration, so the basic call to the cost
function is:
J,J_prime = self.confs[i].parameter.cost(None)
'''
from eoldas_ConfFile import ConfFile
from eoldas_Lib import sortopt
name = name or thisname
if name == None:
import time
thistime = str(time.time())
name = type(self).__name__
name = "%s.%s" % (name,thistime)
self.thisname = name
self.confs = confs
self.top = sortopt(self,'top',ParamStorage())
self.top.general = sortopt(self.top,'general',ParamStorage())
thisname = sortopt(self.top.general,'name',thisname or self.thisname)
logfile = sortopt(self.top.general,'logfile',logfile or 'log.dat')
logdir = sortopt(self.top.general,'logdir',logfile or 'logs')
datadir = sortopt(self.top.general,'datadir',datadir or ['.'])
self.logger = sortlog(self,logfile,logger or self.confs.logger,\
name=self.thisname,logdir=logdir,debug=True)
n_configs = len(self.confs.infos)
self.confs.root = []
self.have_unc = False
try:
logdir = logdir
except:
logdir = self.top.general.logdir
# first set up parameter
conf = confs.infos[0]
general = conf.general
op = conf.parameter
if not 'parameter' in conf.dict():
raise Exception('No parameter field found in %s item %d'%\
(conf.__doc__,0))
general.is_spectral = sortopt(general,'is_spectral',True)
if not general.is_spectral:
sort_non_spectral_model(op,conf.operator,logger=confs.logger)
general.init_test = sortopt(general,'init_test',False)
confs.logger.info('loading parameter state')
conf.parameter.name = 'Operator'
parameter = eval(op.name)(op,general,\
parameter=None,\
logger=confs.logger,\
name=name+".parameter",\
datatype=list(conf.parameter.datatypes),\
logdir=logdir,\
logfile=logfile,\
datadir=datadir)
try:
parameter.transform = parameter.options.x.transform
parameter.invtransform = parameter.options.x.invtransform
except:
parameter.transform = parameter.options.x.names
parameter.invtransform = parameter.options.x.names
# we now have access to parameter.x.state, parameter.x.sd etc
# and possibly parameter.y.state etc.
operators = []
for (opname,op) in conf.operator.dict().iteritems():
if opname != 'helper':
#pdb.set_trace()
exec('from eoldas_%s import %s'%(op.name,op.name))
# make sure the data limits and x bounds are the same
# ... inherit from parameter
op.limits = parameter.options.limits
if not 'datatypes' in op.dict():
op.datatypes = 'x'
thisop = eval(op.name)(op,general,parameter=parameter,\
logger=confs.logger,\
name=name+".%s-%s"%(thisname,opname),\
datatype=list(op.datatypes),\
logdir=logdir,\
logfile=logfile,\
datadir=datadir)
# load from parameter
thisop.loader(parameter)
operators.append(thisop)
try:
thisop.transform = parameter.options.x.transform
thisop.invtransform = parameter.options.x.invtransform
except:
thisop.transform = parameter.options.x.names
thisop.invtransform = parameter.options.x.names
thisop.ploaderMask = np.in1d(parameter.x_meta.state,thisop.x_meta.state)
try:
thisop.invtransform = np.array(thisop.invtransform[thisop.ploaderMask])
thisop.transform = np.array(thisop.transform[thisop.ploaderMask])
except:
ww = thisop.ploaderMask
thisop.invtransform = np.array(thisop.transform)[ww]
thisop.transform = np.array(thisop.transform)[ww]
# sort the loaders
parameter.operators = operators
self.confs.root.append(parameter)
# Now we have set up the operators
# try out the cost function
if general.init_test:
self.logger.info('testing cost function calls')
J,J_prime = self.confs.root[0].cost()
self.logger.info('done')
self.confs.root = np.array(self.confs.root)
def _load_rt_library ( self, rt_library ):
"""
A method that loads up the compiled RT library code and sets some
configuration options. This method tries to import all the methods and
make them available through `self.rt_library.<method_name>`. It is also
a safe importer: if some functions are not available in the library,
it will provide safe methods from them. Additionally, while importing,
a number of configuration options in the class are also updated or set
to default values.
Parameters
-----------
rt_library : string
This is the name of the library object (.so file) that will be
loaded
"""
from eoldas_Lib import sortopt
import_string = "from %s import " % ( rt_library )
self.logger.debug ("Using %s..." % rt_library )
self.rt_library = sortopt(self,'rt_library',ParamStorage())
# 1. Import main functionality
try:
self.logger.debug ("Loading rt_model")
exec ( import_string + "rt_model" )
self.rt_library.rt_model = rt_model
except ImportError:
self.logger.info(\
"Could not import basic RT functionality: rt_model")
self.logger.info(\
"Check library paths, and whether %s.so is available" % \
rt_library)
raise Exception('error importing library %s'%rt_library)
# 1a. Import conditioning methods that can be ignored
try:
exec ( import_string + 'rt_modelpre' )
self.rt_library.rt_modelpre = rt_modelpre
except:
self.rt_library.rt_modelpre = self._nowt
try:
exec ( import_string + 'rt_modelpre' )
self.rt_library.rt_modelpost = rt_modelpre
except:
self.rt_library.rt_modelpost = self._nowt
# 2. Try to import derivative
self.rt_model.ignore_derivative = sortopt(self.rt_model,\
'ignore_derivative',False)
if self.rt_model.ignore_derivative == False:
try:
exec ( import_string + \
"rt_modeld, rt_modeldpre, rt_modeldpost" + \
", rt_modelpred" )
self.rt_model.have_adjoint = True
self.J_prime = self.J_prime_full
self.rt_library.rt_modeld = rt_modeld
self.rt_library.rt_modeldpre = rt_modeldpre
self.rt_library.rt_modeldpost = rt_modeldpost
self.rt_library.rt_modelpred = rt_modelpred
except ImportError:
self.logger.info("No adjoint. Using finite differences approximation.")
self.rt_model.have_adjoint = False
else:
self.logger.info("ignoring adjoint. Using finite differences approximation.")
self.rt_model.have_adjoint = False
self._configure_adjoint ()
try:
exec( import_string + "rt_model_deriv")
self.rt_library.rt_model_deriv = rt_model_deriv
except ImportError:
self.rt_library.rt_model_deriv = None
def reinit(self,options,names=None,datatype=None,limits=None,\
bounds=None,control=None,location=None,env=None,header=None,\
logdir=None,writers={},grid=False,logger=None,\
datadir=None,logfile=None,name=None,info=[],readers=[],debug=None):
'''
Method to re-initialise the class instance
The setup is on the whole controlled by the datatype which
contains e.g. 'x'. This is used to set up the members
self.x and self.y as SpecialVariables (see SpecialVariable
in eoldas_SpecialVariable.py). There are some special attributes
for datatypes starting with 'y'. These are assumed to be
observational data, which means that when they are read, the
data names associated with them are not limited to those in
self.names but rather set to whatever is read in in the data. This
is because the data names for observational data may be terms such
as waveband names etc that need special interpretation. Also,
the default output format for observational data is
different to that of other data.
The elements self.state is a SpecialVariables which means
that they can be assigned various data types (see SpecialVariables)
and loaded accordingly (e.g. if a filename is specified, this is
read in to the data structure. The SpecialVariables contain 'hidden'
datasets, which here are mainly the 'control' and 'location'
information. A SpecialVariable has two internal structures: `data`
and `name`. The former is used to store data values (e.g. the
state values) and the latter to store associated metadata.
For example, `control` is passed here e.g. as [`mask`,`vza`] and this
gives the metadata that are stored in `name`. The actual values
of the control data are stored in the `data` section. For
location, we might be passed [`time`,`row`,`col`], so this is set
in names.location, and the data.location contains the values
of the location at each of these elements. For the actual state
dataset, this is stored according to its name, so for `x` the values
are stored in data.x and the associated data names in name.x.
State datasets must represent at least the mean and standard deviation
of a state for them to be of value in EOLDAS. TThe mean is accessed
as e.g. self.state for the state dataset.
The sd is accessed can be accessed as self._state.sd if it
has been set. This reference can also be used to directly set
data associated with a SpecialVariable, e.g.
self.Data.control = np.zeros([2,3])
to represent 2 samples with 3 control variables. You can access name
information similarly with
print self.Name.control
but this will generate a KeyError if the term has not been set. You
can check it exists with:
key = 'control'
if key in self.Name:
this = (self.Data[key],self.Name[key])
To get e.g. a dictionary representation of a SpecialVariable
you can use eg:
self.Name.to_dict()
to get the name dictionary, or
thisdict = self._state.to_dict()
to get the full representation, which then contains 'data' and
'name' as well as some other information stored in the
SpecialVariable.
You can similarly load them using e.g.
self.Data.update(
ParamStorage().from_dict(thisdict['data'])
combine=True)
'''
# set up a fakes dictionary from the data types
self.set('datatype', datatype)
self.set('fakes', {'state': '_state'})
# first check that options is sensible
self.__check_type(options, ParamStorage, fatal=True)
self.options = options
from eoldas_Lib import set_default_limits,\
check_limits_valid,quantize_location, sortopt
nSpecial = 1
if name == None:
import time
thistime = str(time.time())
name = type(self).__name__
name = "%s.%s" % (name, thistime)
self.thisname = name
self.options.thisname = str(name).replace(' ', '_')
log_terms = {\
'logfile':logfile or sortopt(self.options,'logfile',None),\
'logdir':logdir or sortopt(self.options,'logdir',None),\
'debug' : debug or sortopt(self.options,'debug',True)}
self.datadir = datadir or sortopt(self.options, 'datadir', ["."])
self.header = header or "EOLDAS pickle V1.0 - plewis"
env = env or sortopt(self.options, 'env', None)
names = names or sortopt(self.options, 'names', None)
location = location or sortopt(self.options, 'location', ['time'])
control = control or sortopt(self.options, 'control', [])
limits = limits or sortopt(self.options,'limits',\
set_default_limits(np.array(location)))
limits = limits or self.options.limits
limits = np.array(check_limits_valid(limits))
bounds = bounds or sortopt(self.options,'bounds',\
[[None,None]] * xlen(names))
self.options.bounds = bounds
self.headers = {'PARAMETERS-V2':"PARAMETERS-V2", \
'PARAMETERS':"PARAMETERS", \
'BRDF-UCL':'BRDF-UCL',\
'BRDF': 'BRDF'}
self.headers_2 = {'BRDF-UCL': 'location'}
# The ones pre-loaded are
# self.read_functions = [self.read_pickle,self.read_numpy_fromfile]
self._state = SpecialVariable(info=info,name=self.thisname,\
readers=readers,datadir=self.datadir,\
env=env,writers=writers,\
header=self.header,\
logger=logger,log_terms=log_terms,\
simple=False)
# self._state is where data are read into
# but self.Data and self.Name are where we access them from
self.grid = grid
# this is so we can access this object from
# inside a SpecialVariable
self.state = np.array([0.])
# a default data fmt output
if datatype[0] == 'y':
self.Name.fmt = 'BRDF'
self.Name.state = np.array(['dummy'])
else:
self.Name.fmt = 'PARAMETERS'
n_params = xlen(names)
if not n_params:
error_msg = \
"The field 'names' must be defined in options or"+ \
"passed directly to this method if you have the data type x"
raise Exception(error_msg)
self.Name.state = np.array(names)
self.Name.location = np.array(location)
self.Name.control = np.array(control)
self.Name.header = self.header
self.Name.bounds = np.array(bounds)
self.Name.qlocation = np.array(limits)
self.Name.datadir = datadir
#
# sort this object's name
# sort logging
self.logger = sortlog(self,
log_terms['logfile'],
logger,
name=self.thisname,
logdir=log_terms['logdir'],
debug=log_terms['debug'])
self.logger.info('Initialising %s' % type(self).__name__)
