def preload_prepare(self): ''' Here , we use preload_prepare to make sure the x & any y data are gridded for this operator. This greatly simplifies the application of the differential operator. This method is called before any data are loaded, so ensures they are loaded as a grid. ''' from eoldas_Lib import sortopt for i in np.array(self.options.datatypes).flatten(): # mimic setting the apply_grid flag in options if self.dict().has_key('%s_state' % i): self['%s_state' % i].options[i].apply_grid = True self.novar = sortopt(self, 'novar', False) self.gamma_col = sortopt(self, 'gamma_col', None) self.beenHere = False
def preload_prepare(self): ''' Here , we use preload_prepare to make sure the x & any y data are gridded for this operator. This greatly simplifies the application of the differential operator. This method is called before any data are loaded, so ensures they are loaded as a grid. ''' from eoldas_Lib import sortopt for i in np.array(self.options.datatypes).flatten(): # mimic setting the apply_grid flag in options if self.dict().has_key('%s_state'%i): self['%s_state'%i].options[i].apply_grid = True self.novar = sortopt(self,'novar',False) self.gamma_col = sortopt(self,'gamma_col',None) self.beenHere =False
def sortMask(self): ''' Sort masks for loading and unloading data ''' self.root.options.solve = \ sortopt(self.root.options,\ 'solve',np.array([1]*len(self.root.x_meta.state))) self.solve = np.array(self.root.options.solve).copy() # set up masks for loading and unloading self.mask1 = np.zeros_like(self.root.x.state).astype(bool) ww = np.where(self.solve == 1)[0] self.mask1[...,ww] = True self.mask2 = np.zeros_like(self.root.x.state[0]).astype(bool) ww = np.where(self.solve == 2)[0] self.mask2[ww] = True self.nmask1 = self.mask1.sum() self.nmask2 = self.mask2.sum() self.wmask2 = np.where(self.mask2)[0]
def sortMask(self): ''' Sort masks for loading and unloading data ''' self.root.options.solve = \ sortopt(self.root.options,\ 'solve',np.array([1]*len(self.root.x_meta.state))) self.solve = np.array(self.root.options.solve).copy() # set up masks for loading and unloading self.mask1 = np.zeros_like(self.root.x.state).astype(bool) ww = np.where(self.solve == 1)[0] self.mask1[..., ww] = True self.mask2 = np.zeros_like(self.root.x.state[0]).astype(bool) ww = np.where(self.solve == 2)[0] self.mask2[ww] = True self.nmask1 = self.mask1.sum() self.nmask2 = self.mask2.sum() self.wmask2 = np.where(self.mask2)[0]
def write_output_file(self, filename, name, info=[]): ''' Attempt to write state data as ASCII to filename have a standard interface. Returns: is_error where: is_error = (error,error_msg) The file format is flat ASCII with a header ''' #First check the data is as intended from eoldas_Lib import sortopt try: f = open(filename, 'w') if self.Data.sd != None: sd = self.Data.sd.reshape(self.Data.state.shape) else: # set to unity sd = 0. * self.Data.state + 1.0 except IOError: raise Exception('IOError for %s' % filename) except: try: if 'sd' in self.data.dict(): sd = self.data.sd.reshape(self.data.state.shape) else: # set to unity sd = 0. * self.data.state + 1.0 except: return (True,'Failed to open file %s for writing with call to %s'%\ (filename,str('write_output_file'))) if 'f' in self.__dict__ and f.errors != None: error_msg = str(f.errors) return (True, error_msg) # make a guess at what the format is try: fmt = self.options.result.fmt except: try: fmt = self._state.name.fmt except: try: fmt = self.name.fmt except: fmt = None if fmt == None: fmt = 'PARAMETER' try: state = self.Data.state except: state = self.data.state try: name = self._state.name except: name = self.name gridded = False try: locations = np.array(self.Data.location) except: try: locations = np.array(self.data.location) except: # gridded gridded = True try: controls = np.array(self.Data.control) except: if not gridded: controls = np.array(sortopt(self.data, 'control', [])) if gridded and len(np.atleast_1d(state).shape) == 2: # so no location info try: locations = np.arange(state.shape[0])*name.qlocation[0][2]\ +name.qlocation[0][0] self.data.location = locations except: try: self.Data.location = locations except: # need more complex way to sort grid error_msg = "I can't write this format for this data configuration yet ..." self.logger.error(error_msg) return (True, error_msg) elif gridded: try: (locations, qlocations, state, sd) = self.ungrid(state, sd) except: raise Exception("You are trying to ungrid a dataset that wasn't gridded using State.regrid()" +\ " so the ungridder information is not available. Either load the data using State.grid " +\ " or set it up some other way or avoid calling this method with this type of data") n_samples = np.prod(state.shape) / state.shape[-1] location = np.array(name.location) control = np.array(name.control) try: locations = locations.reshape((n_samples, location.shape[0])) except: # really its gridded then ... # it just thinks it isnt try: locations = self.ungridder.location locations = locations.reshape((n_samples, location.shape[0])) except: raise Exception( 'Inconsistency in data shape ... locations is not consistent with state' ) if len(np.atleast_1d(control)): try: controls = controls.reshape((n_samples, control.shape[0])) except: if (control == np.array(['mask'])).all(): controls = np.ones(n_samples).reshape( (n_samples, control.shape[0])) else: controls = None else: controls = None bands = np.array(name.state) nbands = len(np.atleast_1d(bands)) names = np.atleast_1d(np.array(name.state)) grid = state if fmt == 'BRDF': ''' The 'old' style UCL BRDF format A header of : BRDF n_samples nbands BANDS SD where: BANDS wavebands (names or intervals or wavelength, nm) SD standard deviation (assumed same for all observations) Data columns: time days mask 1 == Good data VZA degrees VAA "" SZA "" SAA "" R1 R2 ... ''' header = "#%s %d %d" % (self.headers[fmt], n_samples, nbands) for i in xrange(nbands): header = header + ' ' + str(names[i]) # for i in xrange(nbands): # header = header + ' ' + str(sd[0,i]) header = header + '\n' ww = np.where(location == 'time')[0] if not len(np.atleast_1d(ww)): location = np.zeros([n_samples, 1]) locations = ['time'] ww = np.where(location == 'time')[0] timer = locations[:, ww] try: ww = np.where(control == 'mask')[0] except: ww = [] if not len(np.atleast_1d(ww)): mask = (0 * timer + 1).astype(int) elif len(np.atleast_1d(control)): mask = controls[:, ww].astype(int) else: mask = (0 * timer + 1).astype(int) try: ww = np.where(control == 'vza')[0] except: ww = [] if not len(np.atleast_1d(ww)): vza = timer * 0 else: vza = controls[:, ww] try: ww = np.where(control == 'sza')[0] except: ww = [] if not len(np.atleast_1d(ww)): sza = timer * 0 else: sza = controls[:, ww] try: ww = np.where(control == 'raa')[0] except: ww = [] if not len(np.atleast_1d(ww)): try: ww1 = np.where(control == 'saa')[0] except: ww1 = [] if not len(np.atleast_1d(ww1)): saa = timer * 0 else: saa = controls[:, ww1] try: ww1 = np.where(control == 'vaa')[0] except: ww1 = [] if not len(np.atleast_1d(ww1)): vaa = timer * 0 else: vaa = controls[:, ww1] else: vaa = controls[:, ww] saa = timer * 0 todo = np.array([timer, mask, vza, vaa, sza, saa]).reshape(6, n_samples).T todo = np.hstack((todo, grid)) elif fmt == 'BRDF-UCL': ''' a slighly modified BRDF output format A header of : #BRDF-UCL n_samples nbands BANDS SD where: BANDS wavebands (names or intervals or wavelength, nm) SD standard deviation A 2nd header line of: #location time row col or similar n_location location fields. Data columns: LOCATION mask 1 == Good data VZA degrees VAA "" SZA "" SAA "" R1 R2 ... S1 S2 ... where LOCATION is n_location columns corresponding to the information in the #location line. R refers ro reflectance/radiance samples and S to SD ''' header = "#%s %d %d" % (self.headers[fmt], n_samples, nbands) header1 = "#%s " % self.headers_2[fmt] for i in xrange(nbands): header = header + ' ' + str(names[i]) #for i in xrange(nbands): # header = header + ' ' + str(sd[0,i]) header = header + '\n' for i in xrange(len(np.atleast_1d(location))): header1 = header1 + ' ' + location[i] header1 = header1 + '\n' header = header + header1 #ww = np.where(location == 'time')[0] #if not len(ww): #location = np.zeros([n_samples,1]) #locations = ['time'] #ww = np.where(location == 'time')[0] #timer = locations[:,ww] ww = np.where(control == 'mask')[0] if not len(np.atleast_1d(ww)): mask = (0 * timer + 1).astype(int) else: mask = controls[:, ww].astype(int) ww = np.where(control == 'vza')[0] if not len(np.atleast_1d(ww)): vza = timer * 0 else: vza = controls[:, ww] ww = np.where(control == 'sza')[0] if not len(np.atleast_1d(ww)): sza = timer * 0 else: sza = controls[:, ww] ww = np.where(control == 'raa')[0] if not len(np.atleast_1d(ww)): ww1 = np.where(control == 'saa')[0] if not len(np.atleast_1d(ww1)): saa = timer * 0 else: saa = controls[:, ww1] ww1 = np.where(control == 'vaa')[0] if not len(np.atleast_1d(ww1)): vaa = timer * 0 else: vaa = controls[:, ww1] else: vaa = controls[:, ww] saa = timer * 0 todo = np.array([mask, vza, vaa, sza, saa]).reshape(5, n_samples).T todo = np.hstack((locations, todo)) todo = np.hstack((todo, grid)) todo = np.hstack((todo, sd)) elif fmt == 'PARAMETERS': ndim = len(np.atleast_1d(location)) header = '#%s' % self.headers[fmt] for i in xrange(ndim): header = header + ' ' + location[i] ndim = len(np.atleast_1d(control)) for i in xrange(ndim): header = header + ' ' + control[i] for i in xrange(names.size): header = header + ' ' + names[i] for i in xrange(names.size): header = header + ' ' + 'sd-%s' % names[i] header = header + '\n' if len(np.atleast_1d(control)): todo = np.hstack((locations, controls)) else: todo = locations grid = grid.reshape( np.array(grid.shape).prod() / grid.shape[-1], grid.shape[-1]) sd = sd.reshape( np.array(grid.shape).prod() / grid.shape[-1], grid.shape[-1]) todo = np.hstack((todo, grid)) todo = np.hstack((todo, sd)) elif fmt == 'PARAMETERS-V2': # dummy ndim = len(np.atleast_1d(location)) header = '#%s' % self.headers[fmt] for i in xrange(ndim): header = header + ' ' + location[i] for i in xrange(names.size): header = header + ' ' + names[i] header = header + '\n' todo = np.hstack((locations, grid)) else: f.close() error_msg = "Unrecognised format for data in write_output_file: %s"\ %fmt return (True, error_msg) # bit complex, but done because cant dump header data # write the header to f (one or two lines) import tempfile f.write(header) # open a tmp file ff2 = tempfile.NamedTemporaryFile() # save the data to the tmp file np.savetxt(ff2.name, todo, fmt=('%11.6f')) # read in the formatted data as lines & write straight after the header fa = open(ff2.name) ff2.close() f.writelines(fa.readlines()) # close both files ... I dont know if you can close the tmp file earlier # to ensure flushed output ...? f.close() return (False,'Data written to %s with %s'% (filename,str \ ('write_output_file')))
def reinit(self,options,names=None,datatype=None,limits=None,\ bounds=None,control=None,location=None,env=None,header=None,\ logdir=None,writers={},grid=False,logger=None,\ datadir=None,logfile=None,name=None,info=[],readers=[],debug=None): ''' Method to re-initialise the class instance The setup is on the whole controlled by the datatype which contains e.g. 'x'. This is used to set up the members self.x and self.y as SpecialVariables (see SpecialVariable in eoldas_SpecialVariable.py). There are some special attributes for datatypes starting with 'y'. These are assumed to be observational data, which means that when they are read, the data names associated with them are not limited to those in self.names but rather set to whatever is read in in the data. This is because the data names for observational data may be terms such as waveband names etc that need special interpretation. Also, the default output format for observational data is different to that of other data. The elements self.state is a SpecialVariables which means that they can be assigned various data types (see SpecialVariables) and loaded accordingly (e.g. if a filename is specified, this is read in to the data structure. The SpecialVariables contain 'hidden' datasets, which here are mainly the 'control' and 'location' information. A SpecialVariable has two internal structures: `data` and `name`. The former is used to store data values (e.g. the state values) and the latter to store associated metadata. For example, `control` is passed here e.g. as [`mask`,`vza`] and this gives the metadata that are stored in `name`. The actual values of the control data are stored in the `data` section. For location, we might be passed [`time`,`row`,`col`], so this is set in names.location, and the data.location contains the values of the location at each of these elements. For the actual state dataset, this is stored according to its name, so for `x` the values are stored in data.x and the associated data names in name.x. State datasets must represent at least the mean and standard deviation of a state for them to be of value in EOLDAS. TThe mean is accessed as e.g. self.state for the state dataset. The sd is accessed can be accessed as self._state.sd if it has been set. This reference can also be used to directly set data associated with a SpecialVariable, e.g. self.Data.control = np.zeros([2,3]) to represent 2 samples with 3 control variables. You can access name information similarly with print self.Name.control but this will generate a KeyError if the term has not been set. You can check it exists with: key = 'control' if key in self.Name: this = (self.Data[key],self.Name[key]) To get e.g. a dictionary representation of a SpecialVariable you can use eg: self.Name.to_dict() to get the name dictionary, or thisdict = self._state.to_dict() to get the full representation, which then contains 'data' and 'name' as well as some other information stored in the SpecialVariable. You can similarly load them using e.g. self.Data.update( ParamStorage().from_dict(thisdict['data']) combine=True) ''' # set up a fakes dictionary from the data types self.set('datatype',datatype) self.set('fakes', {'state':'_state'}) # first check that options is sensible self.__check_type(options,ParamStorage,fatal=True) self.options = options from eoldas_Lib import set_default_limits,\ check_limits_valid,quantize_location, sortopt nSpecial = 1 if name == None: import time thistime = str(time.time()) name = type(self).__name__ name = "%s.%s" % (name,thistime) self.thisname = name self.options.thisname = str(name).replace(' ','_') log_terms = {\ 'logfile':logfile or sortopt(self.options,'logfile',None),\ 'logdir':logdir or sortopt(self.options,'logdir',None),\ 'debug' : debug or sortopt(self.options,'debug',True)} self.datadir = datadir or sortopt(self.options,'datadir',["."]) self.header = header or "EOLDAS pickle V1.0 - plewis" env = env or sortopt(self.options,'env',None) names = names or sortopt(self.options,'names',None) location = location or sortopt(self.options,'location',['time']) control = control or sortopt(self.options,'control',[]) limits = limits or sortopt(self.options,'limits',\ set_default_limits(np.array(location))) limits = limits or self.options.limits limits = np.array(check_limits_valid(limits)) bounds = bounds or sortopt(self.options,'bounds',\ [[None,None]] * xlen(names)) self.options.bounds = bounds self.headers = {'PARAMETERS-V2':"PARAMETERS-V2", \ 'PARAMETERS':"PARAMETERS", \ 'BRDF-UCL':'BRDF-UCL',\ 'BRDF': 'BRDF'} self.headers_2 = {'BRDF-UCL':'location'} # The ones pre-loaded are # self.read_functions = [self.read_pickle,self.read_numpy_fromfile] self._state = SpecialVariable(info=info,name=self.thisname,\ readers=readers,datadir=self.datadir,\ env=env,writers=writers,\ header=self.header,\ logger=logger,log_terms=log_terms,\ simple=False) # self._state is where data are read into # but self.Data and self.Name are where we access them from self.grid=grid # this is so we can access this object from # inside a SpecialVariable self.state = np.array([0.]) # a default data fmt output if datatype[0] == 'y': self.Name.fmt = 'BRDF' self.Name.state = np.array(['dummy']) else: self.Name.fmt = 'PARAMETERS' n_params = xlen(names) if not n_params: error_msg = \ "The field 'names' must be defined in options or"+ \ "passed directly to this method if you have the data type x" raise Exception(error_msg) self.Name.state = np.array(names) self.Name.location = np.array(location) self.Name.control = np.array(control) self.Name.header = self.header self.Name.bounds = np.array(bounds) self.Name.qlocation = np.array(limits) self.Name.datadir = datadir # # sort this object's name # sort logging self.logger = sortlog(self,log_terms['logfile'],logger,name=self.thisname, logdir=log_terms['logdir'],debug=log_terms['debug']) self.logger.info('Initialising %s' % type(self).__name__)
def write_output_file(self,filename,name,info=[]): ''' Attempt to write state data as ASCII to filename have a standard interface. Returns: is_error where: is_error = (error,error_msg) The file format is flat ASCII with a header ''' #First check the data is as intended from eoldas_Lib import sortopt try: f = open(filename,'w') if self.Data.sd != None: sd = self.Data.sd.reshape(self.Data.state.shape) else: # set to unity sd = 0.*self.Data.state + 1.0 except IOError: raise Exception('IOError for %s'%filename) except: try: if 'sd' in self.data.dict(): sd = self.data.sd.reshape(self.data.state.shape) else: # set to unity sd = 0.*self.data.state + 1.0 except: return (True,'Failed to open file %s for writing with call to %s'%\ (filename,str('write_output_file'))) if 'f' in self.__dict__ and f.errors != None: error_msg = str(f.errors) return (True,error_msg) # make a guess at what the format is try: fmt = self.options.result.fmt except: try: fmt = self._state.name.fmt except: try: fmt = self.name.fmt except: fmt = None if fmt == None: fmt = 'PARAMETER' try: state = self.Data.state except: state = self.data.state try: name = self._state.name except: name = self.name gridded = False try: locations = np.array(self.Data.location) except: try: locations = np.array(self.data.location) except: # gridded gridded = True try: controls = np.array(self.Data.control) except: if not gridded: controls = np.array(sortopt(self.data,'control',[])) if gridded and len(np.atleast_1d(state).shape) == 2: # so no location info try: locations = np.arange(state.shape[0])*name.qlocation[0][2]\ +name.qlocation[0][0] self.data.location = locations except: try: self.Data.location = locations except: # need more complex way to sort grid error_msg = "I can't write this format for this data configuration yet ..." self.logger.error(error_msg) return (True,error_msg) elif gridded: try: (locations,qlocations,state,sd) = self.ungrid(state,sd) except: raise Exception("You are trying to ungrid a dataset that wasn't gridded using State.regrid()" +\ " so the ungridder information is not available. Either load the data using State.grid " +\ " or set it up some other way or avoid calling this method with this type of data") n_samples = np.prod(state.shape)/state.shape[-1] location = np.array(name.location) control = np.array(name.control) try: locations = locations.reshape((n_samples,location.shape[0])) except: # really its gridded then ... # it just thinks it isnt try: locations = self.ungridder.location locations = locations.reshape((n_samples,location.shape[0])) except: raise Exception('Inconsistency in data shape ... locations is not consistent with state') if len(np.atleast_1d(control)): try: controls = controls.reshape((n_samples,control.shape[0])) except: if (control == np.array(['mask'])).all(): controls = np.ones(n_samples).reshape((n_samples,control.shape[0])) else: controls = None else: controls = None bands = np.array(name.state) nbands = len(np.atleast_1d(bands)) names = np.atleast_1d(np.array(name.state)) grid = state if fmt == 'BRDF': ''' The 'old' style UCL BRDF format A header of : BRDF n_samples nbands BANDS SD where: BANDS wavebands (names or intervals or wavelength, nm) SD standard deviation (assumed same for all observations) Data columns: time days mask 1 == Good data VZA degrees VAA "" SZA "" SAA "" R1 R2 ... ''' header = "#%s %d %d"%(self.headers[fmt],n_samples,nbands) for i in xrange(nbands): header = header + ' ' + str(names[i]) # for i in xrange(nbands): # header = header + ' ' + str(sd[0,i]) header = header + '\n' ww = np.where(location == 'time')[0] if not len(np.atleast_1d(ww)): location = np.zeros([n_samples,1]) locations = ['time'] ww = np.where(location == 'time')[0] timer = locations[:,ww] try: ww = np.where(control == 'mask')[0] except: ww = [] if not len(np.atleast_1d(ww)): mask = (0*timer+1).astype(int) elif len(np.atleast_1d(control)): mask = controls[:,ww].astype(int) else: mask = (0*timer+1).astype(int) try: ww = np.where(control== 'vza')[0] except: ww = [] if not len(np.atleast_1d(ww)): vza = timer*0 else: vza = controls[:,ww] try: ww = np.where(control== 'sza')[0] except: ww = [] if not len(np.atleast_1d(ww)): sza = timer*0 else: sza = controls[:,ww] try: ww = np.where(control== 'raa')[0] except: ww = [] if not len(np.atleast_1d(ww)): try: ww1 = np.where(control== 'saa')[0] except: ww1 = [] if not len(np.atleast_1d(ww1)): saa = timer*0 else: saa = controls[:,ww1] try: ww1 = np.where(control== 'vaa')[0] except: ww1 = [] if not len(np.atleast_1d(ww1)): vaa = timer*0 else: vaa = controls[:,ww1] else: vaa = controls[:,ww] saa = timer*0 todo = np.array([timer,mask,vza,vaa,sza,saa]).reshape(6,n_samples).T todo = np.hstack((todo,grid)) elif fmt == 'BRDF-UCL': ''' a slighly modified BRDF output format A header of : #BRDF-UCL n_samples nbands BANDS SD where: BANDS wavebands (names or intervals or wavelength, nm) SD standard deviation A 2nd header line of: #location time row col or similar n_location location fields. Data columns: LOCATION mask 1 == Good data VZA degrees VAA "" SZA "" SAA "" R1 R2 ... S1 S2 ... where LOCATION is n_location columns corresponding to the information in the #location line. R refers ro reflectance/radiance samples and S to SD ''' header = "#%s %d %d"%(self.headers[fmt],n_samples,nbands) header1 = "#%s "%self.headers_2[fmt] for i in xrange(nbands): header = header + ' ' + str(names[i]) #for i in xrange(nbands): # header = header + ' ' + str(sd[0,i]) header = header + '\n' for i in xrange(len(np.atleast_1d(location))): header1 = header1 + ' ' + location[i] header1 = header1 + '\n' header = header + header1 #ww = np.where(location == 'time')[0] #if not len(ww): #location = np.zeros([n_samples,1]) #locations = ['time'] #ww = np.where(location == 'time')[0] #timer = locations[:,ww] ww = np.where(control == 'mask')[0] if not len(np.atleast_1d(ww)): mask = (0*timer+1).astype(int) else: mask = controls[:,ww].astype(int) ww = np.where(control== 'vza')[0] if not len(np.atleast_1d(ww)): vza = timer*0 else: vza = controls[:,ww] ww = np.where(control== 'sza')[0] if not len(np.atleast_1d(ww)): sza = timer*0 else: sza = controls[:,ww] ww = np.where(control== 'raa')[0] if not len(np.atleast_1d(ww)): ww1 = np.where(control== 'saa')[0] if not len(np.atleast_1d(ww1)): saa = timer*0 else: saa = controls[:,ww1] ww1 = np.where(control== 'vaa')[0] if not len(np.atleast_1d(ww1)): vaa = timer*0 else: vaa = controls[:,ww1] else: vaa = controls[:,ww] saa = timer*0 todo = np.array([mask,vza,vaa,sza,saa]).reshape(5,n_samples).T todo = np.hstack((locations,todo)) todo = np.hstack((todo,grid)) todo = np.hstack((todo,sd)) elif fmt == 'PARAMETERS': ndim = len(np.atleast_1d(location)) header = '#%s'%self.headers[fmt] for i in xrange(ndim): header = header + ' ' + location[i] ndim = len(np.atleast_1d(control)) for i in xrange(ndim): header = header + ' ' + control[i] for i in xrange(names.size): header = header + ' ' + names[i] for i in xrange(names.size): header = header + ' ' + 'sd-%s'%names[i] header = header + '\n' if len(np.atleast_1d(control)): todo = np.hstack((locations,controls)) else: todo = locations grid = grid.reshape(np.array(grid.shape).prod()/grid.shape[-1],grid.shape[-1]) sd = sd.reshape(np.array(grid.shape).prod()/grid.shape[-1],grid.shape[-1]) todo = np.hstack((todo,grid)) todo = np.hstack((todo,sd)) elif fmt == 'PARAMETERS-V2': # dummy ndim = len(np.atleast_1d(location)) header = '#%s'%self.headers[fmt] for i in xrange(ndim): header = header + ' ' + location[i] for i in xrange(names.size): header = header + ' ' + names[i] header = header + '\n' todo = np.hstack((locations,grid)) else: f.close() error_msg = "Unrecognised format for data in write_output_file: %s"\ %fmt return (True,error_msg) # bit complex, but done because cant dump header data # write the header to f (one or two lines) import tempfile f.write(header) # open a tmp file ff2 = tempfile.NamedTemporaryFile() # save the data to the tmp file np.savetxt(ff2.name,todo,fmt=('%11.6f')) # read in the formatted data as lines & write straight after the header fa = open(ff2.name) ff2.close() f.writelines(fa.readlines()) # close both files ... I dont know if you can close the tmp file earlier # to ensure flushed output ...? f.close() return (False,'Data written to %s with %s'% (filename,str \ ('write_output_file')))
def prep(self,thisconf): ''' A method to prepare the solver ''' self.root = self.confs.root[thisconf] root = self.confs.root[thisconf] self.sortMask() self.op = sortopt(root.general,'optimisation',ParamStorage()) self.op.plot = sortopt(self.op,'plot',0) self.op.name = sortopt(self.op,'name','solver') self.op.maxfunevals = sortopt(self.op,'maxfunevals',2e4) self.op.maxiter = sortopt(self.op,'maxiter',1e4) self.op.gtol = sortopt(self.op,'gtol',1e-3) self.op.iprint = sortopt(self.op,'iprint',1) self.op.solverfn = sortopt(self.op,'solverfn','scipy_lbfgsb') self.op.randomise = sortopt(self.op,'randomise',False) self.op.no_df = sortopt(self.op,'no_df',False) self.result = sortopt(root.options,'result',ParamStorage()) self.result.filename = sortopt(root.options.result,'filename',\ 'results.pkl') self.result.fmt = sortopt(root.options.result,'format','pickle') try: self.transform = self.root.options.x.transform self.invtransform = self.root.options.x.transform except: self.transform = None self.invtransform = None # descend into the operators and identify any observation ops # we then want to be able to write out files (or plot data) # that are H(x). # We only do this for Operators that have both 'x' and 'y' # terms as we store the filename under 'y.result' self.Hx_ops = [] for i,op in enumerate(root.operators): op.loader(root) if 'y_meta' in op.dict() and op.options.y.datatype == 'y': # There is a potential observation op.y_state.options.y.result = \ sortopt(op.y_state.options.y,'result',ParamStorage()) op.y_state.options.y.result.filename = \ sortopt(op.y_state.options.y.result,\ 'filename',op.y_state.thisname) op.y_state.options.y.result.format = \ sortopt(op.y_state.options.y.result,\ 'format','PARAMETERS') state = op.y_state._state this = { \ 'filename':op.y_state.options.y.result.filename,\ 'format':op.y_state.options.y.result.format,\ 'state':state,\ 'y':op.y_state,\ 'op':op,\ 'transform':self.transform,\ 'invtransform':self.invtransform,\ 'Hx':op.linear.H} op.Hx_info = this self.Hx_ops.append(this) else: this = { \ 'transform':self.transform,\ 'invtransform':self.invtransform,\ } op.Hx_info = this
def __init__(self,options,name=None): ''' The class initialiser This sets up the problem, i.e. loads the spectral information from an options structure. The options structure will contain: y_meta.names: -------------------- The names of the state vector elements for a y State as string list. These are interpreted as wavebands associated with the y state. They may be: 1. the names of bandpass functions e.g. MODIS-b2 2. Wavelength intervals e.g. "450-550: 3. Single wavelength e.g. 451 In the case of 2 or 3, we interpret the wavelengths directly from the string. For defined bandpass functions we look in a library to interpret the functions. Optional: options.operator_spectral -------------------- (lmin,lmax,lstep) for the operator. If this is *not* specified then the model is assumed to be 'non spectral' and a flag self.is_spectral is set False. In this case, we form a new set of state variables to solve for and don't bother setting up all of the bandpass information. This means that we have to inform the parameters operator to set up a new set of state vectors. These are based on the bandpass names self.bandnames and the names in options.x_meta.names. options.datadir -------------------- Data directory for bandpass files options.bandpass_libraries -------------------- A list of filenames containing band pass libraries options.bandpass_library -------------------- A dictionary of existing bandpass functions This makes calls to: self.setup_spectral_config() self.load_bandpass_library() self.load_bandpass_from_pulses() self.normalise_wavebands() ''' from eoldas_Lib import isfloat,sortopt if name == None: import time thistime = str(time.time()) name = type(self).__name__ name = "%s.%s" % (name,thistime) self.thisname = name self.options = options self.logger = logging.getLogger (self.thisname+'.spectral') # set self.nlw, self.bandnames & associated size terms # we need this to discretise the bandpass functions self.setup_spectral_config(self.options.y_meta.state) self.is_spectral=sortopt(options.general,'is_spectral',True) if not self.is_spectral: self.logger.info("Non spectral operator specified") return self.logger.info("Spectral operator specified") if 'datadir' in self.options.dict(): self.datadir = self.options.datadir else: self.datadir = ['.','~/.eoldas','..'] self.logger.info("Data directories: %s"%str(self.datadir)) if 'bandpass_library' in self.options.dict(): self.logger.info("Receiving existing bandpass library") self.bandpass_library = self.options.bandpass_library else: self.logger.info("Starting new bandpass library") self.bandpass_library={} if 'bandpass_libraries' in self.options.dict(): self.bandpass_libraries = self.options.bandpass_libraries for i in self.bandpass_libraries: self.logger.info("Attempting to load bandpass library %s"%i) self.bandpass_library,is_error = \ self.load_bandpass_library(i,self.bandpass_library) if is_error[0]: self.logger.error\ ('Cannot load file %s loaded into bandpass library'%i) self.logger.error(is_error[1]) else: self.logger.info("No bandpass libraries specified") self.lstep = sortopt(self,'lstep',1) self.nlw = sortopt(self,'nlw',None) # if you havent defined the wavelength range, use 0-10000 if self.nlw == None: self.logger.info("No wavelength bounds defined") self.logger.info("setting as [0,10000,1]") self.nlw = np.arange(10000) self.pseudowavelengths = [] # Now we can start to load the band names for i in self.bandnames: # check to see if its in the library: if not i in self.bandpass_library: # try to interpret it ok,this = isfloat(i) if ok: self.bandpass_library, bandpass_name = \ self.load_bandpass_from_pulses(str(i),\ this,0.5*self.lstep,self.nlw,\ self.bandpass_library,\ self.lstep) else: # Try splitting it that = i.split('-') if len(that) == 2 and isfloat(that[0])[0] \ and isfloat(that[1])[0]: f0 = float(that[0]) f1 = float(that[1]) self.bandpass_library, bandpass_name = \ self.load_bandpass_from_pulses(i,\ 0.5*(f1+f0),0.5*(f1-f0),self.nlw,\ self.bandpass_library,\ self.lstep) else: f0 = len(self.pseudowavelengths) + self.nlw[0] bandpass_name = i self.logger.info("******************************************************") self.logger.info("requested waveband %s is not in the spectral library"%i) self.logger.info("and cannot be interpreted as a wavelength or wavelength range") self.logger.info("so we will set it up as a pseudowavelength and hope that") self.logger.info("any operator you use does not need to interpret its value") self.logger.info("for reference:") self.logger.info(" band %s"%i) self.logger.info("is interpreted here as:") self.bandpass_library, dummy = \ self.load_bandpass_from_pulses(str(f0),\ f0,self.lstep*0.5,self.nlw,\ self.bandpass_library,\ self.lstep) self.logger.info(" %d nm"%f0) self.logger.info("******************************************************") self.pseudowavelengths.append(i) try: self.bandpass_library[i] = self.bandpass_library[dummy] except: self.bandpass_library[i] = self.bandpass_library[str(f0)] # Now post-process the bandpass library self.normalise_wavebands(self.bandnames)
def _load_rt_library(self, rt_library): """ A method that loads up the compiled RT library code and sets some configuration options. This method tries to import all the methods and make them available through `self.rt_library.<method_name>`. It is also a safe importer: if some functions are not available in the library, it will provide safe methods from them. Additionally, while importing, a number of configuration options in the class are also updated or set to default values. Parameters ----------- rt_library : string This is the name of the library object (.so file) that will be loaded """ from eoldas_Lib import sortopt import_string = "from %s import " % (rt_library) self.logger.debug("Using %s..." % rt_library) self.rt_library = sortopt(self, 'rt_library', ParamStorage()) # 1. Import main functionality try: self.logger.debug("Loading rt_model") exec(import_string + "rt_model") self.rt_library.rt_model = rt_model except ImportError: self.logger.info(\ "Could not import basic RT functionality: rt_model") self.logger.info(\ "Check library paths, and whether %s.so is available" % \ rt_library) raise Exception('error importing library %s' % rt_library) # 1a. Import conditioning methods that can be ignored try: exec(import_string + 'rt_modelpre') self.rt_library.rt_modelpre = rt_modelpre except: self.rt_library.rt_modelpre = self._nowt try: exec(import_string + 'rt_modelpre') self.rt_library.rt_modelpost = rt_modelpre except: self.rt_library.rt_modelpost = self._nowt # 2. Try to import derivative self.rt_model.ignore_derivative = sortopt(self.rt_model,\ 'ignore_derivative',False) if self.rt_model.ignore_derivative == False: try: exec ( import_string + \ "rt_modeld, rt_modeldpre, rt_modeldpost" + \ ", rt_modelpred" ) self.rt_model.have_adjoint = True self.J_prime = self.J_prime_full self.rt_library.rt_modeld = rt_modeld self.rt_library.rt_modeldpre = rt_modeldpre self.rt_library.rt_modeldpost = rt_modeldpost self.rt_library.rt_modelpred = rt_modelpred except ImportError: self.logger.info( "No adjoint. Using finite differences approximation.") self.rt_model.have_adjoint = False else: self.logger.info( "ignoring adjoint. Using finite differences approximation.") self.rt_model.have_adjoint = False self._configure_adjoint() try: exec(import_string + "rt_model_deriv") self.rt_library.rt_model_deriv = rt_model_deriv except ImportError: self.rt_library.rt_model_deriv = None
def prep(self, thisconf): ''' A method to prepare the solver ''' self.root = self.confs.root[thisconf] root = self.confs.root[thisconf] self.sortMask() self.op = sortopt(root.general, 'optimisation', ParamStorage()) self.op.plot = sortopt(self.op, 'plot', 0) self.op.name = sortopt(self.op, 'name', 'solver') self.op.maxfunevals = sortopt(self.op, 'maxfunevals', 2e4) self.op.maxiter = sortopt(self.op, 'maxiter', 1e4) self.op.gtol = sortopt(self.op, 'gtol', 1e-3) self.op.iprint = sortopt(self.op, 'iprint', 1) self.op.solverfn = sortopt(self.op, 'solverfn', 'scipy_lbfgsb') self.op.randomise = sortopt(self.op, 'randomise', False) self.op.no_df = sortopt(self.op, 'no_df', False) self.result = sortopt(root.options, 'result', ParamStorage()) self.result.filename = sortopt(root.options.result,'filename',\ 'results.pkl') self.result.fmt = sortopt(root.options.result, 'format', 'pickle') try: self.transform = self.root.options.x.transform self.invtransform = self.root.options.x.transform except: self.transform = None self.invtransform = None # descend into the operators and identify any observation ops # we then want to be able to write out files (or plot data) # that are H(x). # We only do this for Operators that have both 'x' and 'y' # terms as we store the filename under 'y.result' self.Hx_ops = [] for i, op in enumerate(root.operators): op.loader(root) if 'y_meta' in op.dict() and op.options.y.datatype == 'y': # There is a potential observation op.y_state.options.y.result = \ sortopt(op.y_state.options.y,'result',ParamStorage()) op.y_state.options.y.result.filename = \ sortopt(op.y_state.options.y.result,\ 'filename',op.y_state.thisname) op.y_state.options.y.result.format = \ sortopt(op.y_state.options.y.result,\ 'format','PARAMETERS') state = op.y_state._state this = { \ 'filename':op.y_state.options.y.result.filename,\ 'format':op.y_state.options.y.result.format,\ 'state':state,\ 'y':op.y_state,\ 'op':op,\ 'transform':self.transform,\ 'invtransform':self.invtransform,\ 'Hx':op.linear.H} op.Hx_info = this self.Hx_ops.append(this) else: this = { \ 'transform':self.transform,\ 'invtransform':self.invtransform,\ } op.Hx_info = this
def __init__(self, confs, logger=None, logfile=None, thisname=None, name=None, datadir=None, logdir=None): ''' Initialise the solver. This does the following: 1. Read configuration file(s) 2. Load operators 3. Test the call to the cost function There can be multiple groups of configuration files, so self.confs, that holds the core information setup here can contain multiple configurations. The number of configurations is len(confs.infos) and the ith configuration is conf = self.confs.infos[i]. Various loggers are available throughout the classes used, but the top level logger is self.confs.logger, so you can log with e.g. self.confs.logger.info('this is some info') The root operator is stored in self.confs.root[i] for the ith configuration, so the basic call to the cost function is: J,J_prime = self.confs[i].parameter.cost(None) ''' from eoldas_ConfFile import ConfFile from eoldas_Lib import sortopt name = name or thisname if name == None: import time thistime = str(time.time()) name = type(self).__name__ name = "%s.%s" % (name, thistime) self.thisname = name self.confs = confs self.top = sortopt(self, 'top', ParamStorage()) self.top.general = sortopt(self.top, 'general', ParamStorage()) thisname = sortopt(self.top.general, 'name', thisname or self.thisname) logfile = sortopt(self.top.general, 'logfile', logfile or 'log.dat') logdir = sortopt(self.top.general, 'logdir', logfile or 'logs') datadir = sortopt(self.top.general, 'datadir', datadir or ['.']) self.logger = sortlog(self,logfile,logger or self.confs.logger,\ name=self.thisname,logdir=logdir,debug=True) n_configs = len(self.confs.infos) self.confs.root = [] self.have_unc = False try: logdir = logdir except: logdir = self.top.general.logdir # first set up parameter conf = confs.infos[0] general = conf.general op = conf.parameter if not 'parameter' in conf.dict(): raise Exception('No parameter field found in %s item %d'%\ (conf.__doc__,0)) general.is_spectral = sortopt(general, 'is_spectral', True) if not general.is_spectral: sort_non_spectral_model(op, conf.operator, logger=confs.logger) general.init_test = sortopt(general, 'init_test', False) confs.logger.info('loading parameter state') conf.parameter.name = 'Operator' parameter = eval(op.name)(op,general,\ parameter=None,\ logger=confs.logger,\ name=name+".parameter",\ datatype=list(conf.parameter.datatypes),\ logdir=logdir,\ logfile=logfile,\ datadir=datadir) try: parameter.transform = parameter.options.x.transform parameter.invtransform = parameter.options.x.invtransform except: parameter.transform = parameter.options.x.names parameter.invtransform = parameter.options.x.names # we now have access to parameter.x.state, parameter.x.sd etc # and possibly parameter.y.state etc. operators = [] for (opname, op) in conf.operator.dict().iteritems(): if opname != 'helper': #pdb.set_trace() exec('from eoldas_%s import %s' % (op.name, op.name)) # make sure the data limits and x bounds are the same # ... inherit from parameter op.limits = parameter.options.limits if not 'datatypes' in op.dict(): op.datatypes = 'x' thisop = eval(op.name)(op,general,parameter=parameter,\ logger=confs.logger,\ name=name+".%s-%s"%(thisname,opname),\ datatype=list(op.datatypes),\ logdir=logdir,\ logfile=logfile,\ datadir=datadir) # load from parameter thisop.loader(parameter) operators.append(thisop) try: thisop.transform = parameter.options.x.transform thisop.invtransform = parameter.options.x.invtransform except: thisop.transform = parameter.options.x.names thisop.invtransform = parameter.options.x.names thisop.ploaderMask = np.in1d(parameter.x_meta.state, thisop.x_meta.state) try: thisop.invtransform = np.array( thisop.invtransform[thisop.ploaderMask]) thisop.transform = np.array( thisop.transform[thisop.ploaderMask]) except: ww = thisop.ploaderMask thisop.invtransform = np.array(thisop.transform)[ww] thisop.transform = np.array(thisop.transform)[ww] # sort the loaders parameter.operators = operators self.confs.root.append(parameter) # Now we have set up the operators # try out the cost function if general.init_test: self.logger.info('testing cost function calls') J, J_prime = self.confs.root[0].cost() self.logger.info('done') self.confs.root = np.array(self.confs.root)
def postload_prepare(self): ''' This is called on initialisation, after data have been read in Here, we load parameters specifically associated with the model H(x). In the case of this differential operator, there are: model_order : order of the differential operator (integer) wraparound : edge conditions Can be: periodic none reflexive lag : The (time/space) lag at which the finite difference is calculated in the differential operator here. If this is 1, then we take the difference between each sample point and its neighbour. This is what we normally use. The main purpose of this mechanism is to allow differences at multiple lags to be calculated (fuller autocorrelation function constraints as in kriging) Multiple lags can be specified (which you could use to perform kriging), in which case lag weight should also be specified. lag_weight : The weight associated with each lag. This will generally be decreasing with increasing lag for a 'usual' autocorrelation function. There is no point specifying this if only a single lag is specified as the function is normalised. If the conditions are specified as periodic the period of the function can also be specified, e.g. for time varying data, you could specify 365 for the periodic period. These are specified in the configuration file as operator.modelt.rt_model.model_order operator.modelt.rt_model.wraparound operator.modelt.rt_model.lag operator.modelt.rt_model.lag_weight The default values (set here) are 1, 'none', 1 and 1 respectively. To specify the period for `periodic` specify e.g.: [operator.modelt.rt_model] wraparound=periodic,365 The default period is set to 0, which implies that it is periodic on whatever the data extent is. Or for multiple lags: [operator.modelt.rt_model] lag=1,2,3,4,5 lag_weight=1,0.7,0.5,0.35,0.2 NB this lag mechanism has not yet been fully tested and should be used with caution. It is intended more as a placeholder for future developments. Finally, we can also decide to work with inverse gamma (i.e. an uncertainty-based measure) This is achieved by setting the flag operator.modelt.rt_model.inverse_gamma=True This flag should be set if you intend to estimate gamma in the Data Assimilation. Again, the is experimental and should be used with caution. ''' from eoldas_Lib import sortopt self.rt_model = sortopt(self.options, 'rt_model', ParamStorage()) self.rt_model.lag = sortopt(self.rt_model, 'lag', 1) self.rt_model.inverse_gamma= \ sortopt(self.rt_model,'inverse_gamma',False) self.rt_model.model_order = \ sortopt(self.rt_model,'model_order',1) self.rt_model.wraparound = \ sortopt(self.rt_model,'wraparound','none') self.rt_model.wraparound_mod = 0 if np.array(self.rt_model.wraparound).size == 2 and \ np.array(self.rt_model.wraparound)[0] == 'periodic': self.rt_model.wraparound_mod = \ np.array(self.rt_model.wraparound)[1] self.rt_model.wraparound = \ np.array(self.rt_model.wraparound)[0] self.rt_model.lag = \ sortopt(self.rt_model,'lag',[1]) self.rt_model.lag = np.array(self.rt_model.lag).flatten() self.rt_model.lag_weight = \ sortopt(self.rt_model,'lag_weight',[1.]*\ self.rt_model.lag.size) self.rt_model.lag_weight = np.array(\ self.rt_model.lag_weight).flatten().astype(float) if self.rt_model.lag_weight.sum() == 0: self.rt_model.lag_weight[:] = np.ones( self.rt_model.lag_weight.size) self.rt_model.lag_weight = self.rt_model.lag_weight\ / self.rt_model.lag_weight.sum()
def postload_prepare(self): ''' This is called on initialisation, after data have been read in Here, we load parameters specifically associated with the model H(x). In the case of this differential operator, there are: model_order : order of the differential operator (integer) wraparound : edge conditions Can be: periodic none reflexive lag : The (time/space) lag at which the finite difference is calculated in the differential operator here. If this is 1, then we take the difference between each sample point and its neighbour. This is what we normally use. The main purpose of this mechanism is to allow differences at multiple lags to be calculated (fuller autocorrelation function constraints as in kriging) Multiple lags can be specified (which you could use to perform kriging), in which case lag weight should also be specified. lag_weight : The weight associated with each lag. This will generally be decreasing with increasing lag for a 'usual' autocorrelation function. There is no point specifying this if only a single lag is specified as the function is normalised. If the conditions are specified as periodic the period of the function can also be specified, e.g. for time varying data, you could specify 365 for the periodic period. These are specified in the configuration file as operator.modelt.rt_model.model_order operator.modelt.rt_model.wraparound operator.modelt.rt_model.lag operator.modelt.rt_model.lag_weight The default values (set here) are 1, 'none', 1 and 1 respectively. To specify the period for `periodic` specify e.g.: [operator.modelt.rt_model] wraparound=periodic,365 The default period is set to 0, which implies that it is periodic on whatever the data extent is. Or for multiple lags: [operator.modelt.rt_model] lag=1,2,3,4,5 lag_weight=1,0.7,0.5,0.35,0.2 NB this lag mechanism has not yet been fully tested and should be used with caution. It is intended more as a placeholder for future developments. Finally, we can also decide to work with inverse gamma (i.e. an uncertainty-based measure) This is achieved by setting the flag operator.modelt.rt_model.inverse_gamma=True This flag should be set if you intend to estimate gamma in the Data Assimilation. Again, the is experimental and should be used with caution. ''' from eoldas_Lib import sortopt self.rt_model = sortopt(self.options,'rt_model',ParamStorage()) self.rt_model.lag = sortopt(self.rt_model,'lag',1) self.rt_model.inverse_gamma= \ sortopt(self.rt_model,'inverse_gamma',False) self.rt_model.model_order = \ sortopt(self.rt_model,'model_order',1) self.rt_model.wraparound = \ sortopt(self.rt_model,'wraparound','none') self.rt_model.wraparound_mod = 0 if np.array(self.rt_model.wraparound).size == 2 and \ np.array(self.rt_model.wraparound)[0] == 'periodic': self.rt_model.wraparound_mod = \ np.array(self.rt_model.wraparound)[1] self.rt_model.wraparound = \ np.array(self.rt_model.wraparound)[0] self.rt_model.lag = \ sortopt(self.rt_model,'lag',[1]) self.rt_model.lag = np.array(self.rt_model.lag).flatten() self.rt_model.lag_weight = \ sortopt(self.rt_model,'lag_weight',[1.]*\ self.rt_model.lag.size) self.rt_model.lag_weight = np.array(\ self.rt_model.lag_weight).flatten().astype(float) if self.rt_model.lag_weight.sum() == 0: self.rt_model.lag_weight[:] = np.ones(self.rt_model.lag_weight.size) self.rt_model.lag_weight = self.rt_model.lag_weight\ / self.rt_model.lag_weight.sum()
def __init__(self,confs,logger=None,logfile=None,thisname=None,name=None,datadir=None,logdir=None): ''' Initialise the solver. This does the following: 1. Read configuration file(s) 2. Load operators 3. Test the call to the cost function There can be multiple groups of configuration files, so self.confs, that holds the core information setup here can contain multiple configurations. The number of configurations is len(confs.infos) and the ith configuration is conf = self.confs.infos[i]. Various loggers are available throughout the classes used, but the top level logger is self.confs.logger, so you can log with e.g. self.confs.logger.info('this is some info') The root operator is stored in self.confs.root[i] for the ith configuration, so the basic call to the cost function is: J,J_prime = self.confs[i].parameter.cost(None) ''' from eoldas_ConfFile import ConfFile from eoldas_Lib import sortopt name = name or thisname if name == None: import time thistime = str(time.time()) name = type(self).__name__ name = "%s.%s" % (name,thistime) self.thisname = name self.confs = confs self.top = sortopt(self,'top',ParamStorage()) self.top.general = sortopt(self.top,'general',ParamStorage()) thisname = sortopt(self.top.general,'name',thisname or self.thisname) logfile = sortopt(self.top.general,'logfile',logfile or 'log.dat') logdir = sortopt(self.top.general,'logdir',logfile or 'logs') datadir = sortopt(self.top.general,'datadir',datadir or ['.']) self.logger = sortlog(self,logfile,logger or self.confs.logger,\ name=self.thisname,logdir=logdir,debug=True) n_configs = len(self.confs.infos) self.confs.root = [] self.have_unc = False try: logdir = logdir except: logdir = self.top.general.logdir # first set up parameter conf = confs.infos[0] general = conf.general op = conf.parameter if not 'parameter' in conf.dict(): raise Exception('No parameter field found in %s item %d'%\ (conf.__doc__,0)) general.is_spectral = sortopt(general,'is_spectral',True) if not general.is_spectral: sort_non_spectral_model(op,conf.operator,logger=confs.logger) general.init_test = sortopt(general,'init_test',False) confs.logger.info('loading parameter state') conf.parameter.name = 'Operator' parameter = eval(op.name)(op,general,\ parameter=None,\ logger=confs.logger,\ name=name+".parameter",\ datatype=list(conf.parameter.datatypes),\ logdir=logdir,\ logfile=logfile,\ datadir=datadir) try: parameter.transform = parameter.options.x.transform parameter.invtransform = parameter.options.x.invtransform except: parameter.transform = parameter.options.x.names parameter.invtransform = parameter.options.x.names # we now have access to parameter.x.state, parameter.x.sd etc # and possibly parameter.y.state etc. operators = [] for (opname,op) in conf.operator.dict().iteritems(): if opname != 'helper': #pdb.set_trace() exec('from eoldas_%s import %s'%(op.name,op.name)) # make sure the data limits and x bounds are the same # ... inherit from parameter op.limits = parameter.options.limits if not 'datatypes' in op.dict(): op.datatypes = 'x' thisop = eval(op.name)(op,general,parameter=parameter,\ logger=confs.logger,\ name=name+".%s-%s"%(thisname,opname),\ datatype=list(op.datatypes),\ logdir=logdir,\ logfile=logfile,\ datadir=datadir) # load from parameter thisop.loader(parameter) operators.append(thisop) try: thisop.transform = parameter.options.x.transform thisop.invtransform = parameter.options.x.invtransform except: thisop.transform = parameter.options.x.names thisop.invtransform = parameter.options.x.names thisop.ploaderMask = np.in1d(parameter.x_meta.state,thisop.x_meta.state) try: thisop.invtransform = np.array(thisop.invtransform[thisop.ploaderMask]) thisop.transform = np.array(thisop.transform[thisop.ploaderMask]) except: ww = thisop.ploaderMask thisop.invtransform = np.array(thisop.transform)[ww] thisop.transform = np.array(thisop.transform)[ww] # sort the loaders parameter.operators = operators self.confs.root.append(parameter) # Now we have set up the operators # try out the cost function if general.init_test: self.logger.info('testing cost function calls') J,J_prime = self.confs.root[0].cost() self.logger.info('done') self.confs.root = np.array(self.confs.root)
def _load_rt_library ( self, rt_library ): """ A method that loads up the compiled RT library code and sets some configuration options. This method tries to import all the methods and make them available through `self.rt_library.<method_name>`. It is also a safe importer: if some functions are not available in the library, it will provide safe methods from them. Additionally, while importing, a number of configuration options in the class are also updated or set to default values. Parameters ----------- rt_library : string This is the name of the library object (.so file) that will be loaded """ from eoldas_Lib import sortopt import_string = "from %s import " % ( rt_library ) self.logger.debug ("Using %s..." % rt_library ) self.rt_library = sortopt(self,'rt_library',ParamStorage()) # 1. Import main functionality try: self.logger.debug ("Loading rt_model") exec ( import_string + "rt_model" ) self.rt_library.rt_model = rt_model except ImportError: self.logger.info(\ "Could not import basic RT functionality: rt_model") self.logger.info(\ "Check library paths, and whether %s.so is available" % \ rt_library) raise Exception('error importing library %s'%rt_library) # 1a. Import conditioning methods that can be ignored try: exec ( import_string + 'rt_modelpre' ) self.rt_library.rt_modelpre = rt_modelpre except: self.rt_library.rt_modelpre = self._nowt try: exec ( import_string + 'rt_modelpre' ) self.rt_library.rt_modelpost = rt_modelpre except: self.rt_library.rt_modelpost = self._nowt # 2. Try to import derivative self.rt_model.ignore_derivative = sortopt(self.rt_model,\ 'ignore_derivative',False) if self.rt_model.ignore_derivative == False: try: exec ( import_string + \ "rt_modeld, rt_modeldpre, rt_modeldpost" + \ ", rt_modelpred" ) self.rt_model.have_adjoint = True self.J_prime = self.J_prime_full self.rt_library.rt_modeld = rt_modeld self.rt_library.rt_modeldpre = rt_modeldpre self.rt_library.rt_modeldpost = rt_modeldpost self.rt_library.rt_modelpred = rt_modelpred except ImportError: self.logger.info("No adjoint. Using finite differences approximation.") self.rt_model.have_adjoint = False else: self.logger.info("ignoring adjoint. Using finite differences approximation.") self.rt_model.have_adjoint = False self._configure_adjoint () try: exec( import_string + "rt_model_deriv") self.rt_library.rt_model_deriv = rt_model_deriv except ImportError: self.rt_library.rt_model_deriv = None
def reinit(self,options,names=None,datatype=None,limits=None,\ bounds=None,control=None,location=None,env=None,header=None,\ logdir=None,writers={},grid=False,logger=None,\ datadir=None,logfile=None,name=None,info=[],readers=[],debug=None): ''' Method to re-initialise the class instance The setup is on the whole controlled by the datatype which contains e.g. 'x'. This is used to set up the members self.x and self.y as SpecialVariables (see SpecialVariable in eoldas_SpecialVariable.py). There are some special attributes for datatypes starting with 'y'. These are assumed to be observational data, which means that when they are read, the data names associated with them are not limited to those in self.names but rather set to whatever is read in in the data. This is because the data names for observational data may be terms such as waveband names etc that need special interpretation. Also, the default output format for observational data is different to that of other data. The elements self.state is a SpecialVariables which means that they can be assigned various data types (see SpecialVariables) and loaded accordingly (e.g. if a filename is specified, this is read in to the data structure. The SpecialVariables contain 'hidden' datasets, which here are mainly the 'control' and 'location' information. A SpecialVariable has two internal structures: `data` and `name`. The former is used to store data values (e.g. the state values) and the latter to store associated metadata. For example, `control` is passed here e.g. as [`mask`,`vza`] and this gives the metadata that are stored in `name`. The actual values of the control data are stored in the `data` section. For location, we might be passed [`time`,`row`,`col`], so this is set in names.location, and the data.location contains the values of the location at each of these elements. For the actual state dataset, this is stored according to its name, so for `x` the values are stored in data.x and the associated data names in name.x. State datasets must represent at least the mean and standard deviation of a state for them to be of value in EOLDAS. TThe mean is accessed as e.g. self.state for the state dataset. The sd is accessed can be accessed as self._state.sd if it has been set. This reference can also be used to directly set data associated with a SpecialVariable, e.g. self.Data.control = np.zeros([2,3]) to represent 2 samples with 3 control variables. You can access name information similarly with print self.Name.control but this will generate a KeyError if the term has not been set. You can check it exists with: key = 'control' if key in self.Name: this = (self.Data[key],self.Name[key]) To get e.g. a dictionary representation of a SpecialVariable you can use eg: self.Name.to_dict() to get the name dictionary, or thisdict = self._state.to_dict() to get the full representation, which then contains 'data' and 'name' as well as some other information stored in the SpecialVariable. You can similarly load them using e.g. self.Data.update( ParamStorage().from_dict(thisdict['data']) combine=True) ''' # set up a fakes dictionary from the data types self.set('datatype', datatype) self.set('fakes', {'state': '_state'}) # first check that options is sensible self.__check_type(options, ParamStorage, fatal=True) self.options = options from eoldas_Lib import set_default_limits,\ check_limits_valid,quantize_location, sortopt nSpecial = 1 if name == None: import time thistime = str(time.time()) name = type(self).__name__ name = "%s.%s" % (name, thistime) self.thisname = name self.options.thisname = str(name).replace(' ', '_') log_terms = {\ 'logfile':logfile or sortopt(self.options,'logfile',None),\ 'logdir':logdir or sortopt(self.options,'logdir',None),\ 'debug' : debug or sortopt(self.options,'debug',True)} self.datadir = datadir or sortopt(self.options, 'datadir', ["."]) self.header = header or "EOLDAS pickle V1.0 - plewis" env = env or sortopt(self.options, 'env', None) names = names or sortopt(self.options, 'names', None) location = location or sortopt(self.options, 'location', ['time']) control = control or sortopt(self.options, 'control', []) limits = limits or sortopt(self.options,'limits',\ set_default_limits(np.array(location))) limits = limits or self.options.limits limits = np.array(check_limits_valid(limits)) bounds = bounds or sortopt(self.options,'bounds',\ [[None,None]] * xlen(names)) self.options.bounds = bounds self.headers = {'PARAMETERS-V2':"PARAMETERS-V2", \ 'PARAMETERS':"PARAMETERS", \ 'BRDF-UCL':'BRDF-UCL',\ 'BRDF': 'BRDF'} self.headers_2 = {'BRDF-UCL': 'location'} # The ones pre-loaded are # self.read_functions = [self.read_pickle,self.read_numpy_fromfile] self._state = SpecialVariable(info=info,name=self.thisname,\ readers=readers,datadir=self.datadir,\ env=env,writers=writers,\ header=self.header,\ logger=logger,log_terms=log_terms,\ simple=False) # self._state is where data are read into # but self.Data and self.Name are where we access them from self.grid = grid # this is so we can access this object from # inside a SpecialVariable self.state = np.array([0.]) # a default data fmt output if datatype[0] == 'y': self.Name.fmt = 'BRDF' self.Name.state = np.array(['dummy']) else: self.Name.fmt = 'PARAMETERS' n_params = xlen(names) if not n_params: error_msg = \ "The field 'names' must be defined in options or"+ \ "passed directly to this method if you have the data type x" raise Exception(error_msg) self.Name.state = np.array(names) self.Name.location = np.array(location) self.Name.control = np.array(control) self.Name.header = self.header self.Name.bounds = np.array(bounds) self.Name.qlocation = np.array(limits) self.Name.datadir = datadir # # sort this object's name # sort logging self.logger = sortlog(self, log_terms['logfile'], logger, name=self.thisname, logdir=log_terms['logdir'], debug=log_terms['debug']) self.logger.info('Initialising %s' % type(self).__name__)