def read_numpy_fromfile(self, goodfile, dataset, info=None): ''' Utility to try to read a file goodfile This simple version of the utility just tries a numpy.fromfile It is not generally recommended to use this first as it will skip headers etc that may contain information However, this is a good illustration of the required interface format for file readers. Inputs: goodfile : a filename (ideally one that exists) dataset : dataset name (e.g. x_state) info : a list of other information Outputs: retval : ParamStorage error : tuple where: retval contains dataset 'this' in data['this'] and other information (e.g. locations etc) in retval.data and retval.names ''' #self.store_header = goodfile.open().readline().close() this = np.genfromtxt(goodfile, skip_header=1) l = len(np.atleast_1d(this).shape) if this.size > 0: retval = ParamStorage() retval.name = ParamStorage() retval.data = ParamStorage() retval.data[dataset] = this retval.name.filename = goodfile retval.name.fmt = 'np.genfromtxt' return retval, (False, "") error = True error_msg = "Failed numpy read of %s" % goodfile return 0, (error, error_msg)
def read_numpy_fromfile(self,goodfile,dataset,info=None): ''' Utility to try to read a file goodfile This simple version of the utility just tries a numpy.fromfile It is not generally recommended to use this first as it will skip headers etc that may contain information However, this is a good illustration of the required interface format for file readers. Inputs: goodfile : a filename (ideally one that exists) dataset : dataset name (e.g. x_state) info : a list of other information Outputs: retval : ParamStorage error : tuple where: retval contains dataset 'this' in data['this'] and other information (e.g. locations etc) in retval.data and retval.names ''' #self.store_header = goodfile.open().readline().close() this = np.genfromtxt(goodfile,skip_header=1) l = len(np.atleast_1d(this).shape) if this.size > 0: retval = ParamStorage() retval.name = ParamStorage() retval.data = ParamStorage() retval.data[dataset] = this retval.name.filename = goodfile retval.name.fmt = 'np.genfromtxt' return retval,(False,"") error = True error_msg = "Failed numpy read of %s"%goodfile return 0,(error,error_msg)
def demonstration(): # set state to a filename # and it will be loaded with the data x = DemoClass() data = {'state':np.ones(2)*5. ,'foo':np.ones(10)} name = {'state':'of the nation','foo':'bar'} this = {'data':data,'name':name} x.state = this print 1,x.state,x['state'] x.oats = 'beans and barley-o' # nothing set so far print 2,x['state'] # should return the same print 3,x.state x.state = 'test/data_type/input/test.brf' print 4,x.state print 5,x.Name.fmt # set state to a dict and # it will load from that data = {'state':np.zeros(10)} name = {'state':'foo'} x.state = {'data':data,'name':name} print 6,x.state # set from a ParamStorage # and it will be loaded this = ParamStorage() this.data = ParamStorage() this.name = ParamStorage() this.data.state = np.ones(10) this.name.state = 'bar' this.data.sd = np.ones(10)*2. this.name.sd = 'sd info' # assign the data x.state = this # access the data print 7,x.state # access another member # Data, Name == implicitly .state print 8,x.Data.sd print 9,x.Name.sd # set directly x.Name.sd = 'bar' print 10,x.Name.sd # set from a tuple (data,name) # or a list [data,name] data = 'foo' name = 'bar' x.state = (data,name) print 11,x.state x.state = [name,data] print 12,x.state # set from a numpy array x.state = np.array(np.arange(10)) print 13,x.state # set from another state y = DemoClass() y.state = x.state x.state = x.state * 2 print 'x state',x.state print 'y state',y.state # set from a float x.state = 100. print 14,x.state # another interesting feature # we have 2 special terms in demonstration # state and other # if we set up some strcture for data # for other this = ParamStorage() this.data = ParamStorage() this.name = ParamStorage() this.data.other = np.ones(10) this.name.other = 'bar' # and the assign it to state x.state = this print 15,'state',x.state # we see state is unchanged # but other is also not set. print 16,'other',x.other # we load into other using: x.other = this print 'other',x.other # but if you look at the information contained print 17,x._other.to_dict() print 18,x._state.to_dict() # or better writtem as: print 19,x.var('state').to_dict() # you will see that state contains the other data that was loaded # a simple way to write out the data is to a pickle # x.write_pickle('xstate','x_state.pkl') # but try to avoid using the underscores print 20,"x state in pickle:",x.state SpecialVariable.write(x._state,'x_state.pkl',fmt='pickle') # which we can reload: z = DemoClass() z.state = 'x_state.pkl' print 21,"z state read from pickle",z.state # which is the same as a forced read ... zz = DemoClass() zz.state = 'x_state.pkl' print 22,zz.state # read a brf file zz.Name.qlocation = [[170,365,1],[0,500,1],[200,200,1]] zz.state = 'test/data_type/input/interpolated_data.npz' print zz.state SpecialVariable.write(zz._state,'test/data_type/output/interpolated_data.pkl',fmt='pickle') zz.state = 'test/data_type/input/test.brf' print zz.state
def read_numpy(self, filename, name, info=[]): ''' Try to read the file as as a NpzFile file ''' from eoldas_Lib import set_default_limits,check_limits_valid,\ quantize_location,dequantize_location # none of these ciritical to functioning try: info = self._state.info except: info = [] try: names = self.name.state except: try: names = self.Name.state except: names = None try: control = self.Name.control except: try: control = self.name.control except: control = None try: location = self.name.location except: try: location = self.Name.location except: location = ['time', 'row', 'col'] try: limits = self.name.qlocation except: try: limits = self.Name.qlocation except: limits = set_default_limits(location) # refl_check=False,names=None,\ # control=['mask','vza','vaa','sza','saa'],\ # location=['time','row','col'],limits=None # location specifies the dimesions and names of the # problem, e.g., & typically [time,row,col] limits = np.array(check_limits_valid(limits)) try: f = np.load(filename) if not type(f).__name__ == 'NpzFile': f.close() self.error_msg = "%s is not a NpzFile" % filename self.error = True if 'logger' in self or 'logger' in self.dict(): self.logger.info(self.error_msg) return 0, (self.error, self.error_msg) except: self.error_msg = "a problem opening %s as a NpzFile" % filename self.error = True if 'logger' in self or 'logger' in self.dict(): self.logger.info(self.error_msg) return 0, (self.error, self.error_msg) # ok so far then # lets have a look inside ncontents = np.array(f.files) contents = np.array(f.files) # translation table for default names def_names = 'b1 b2 b3 b4 b5 b6 b7'.split() if names == None: # assume MODIS names = def_names def_alt_names = \ '645.5 856.5 465.6 553.6 1241.6 1629.1 2114.1'.split() # look for any of names in contents datasets = [] alt_datasets = [] alt_names = names for i in xrange(len(np.atleast_1d(contents))): if contents[i] in names: datasets.append(i) if not len(np.atleast_1d(datasets)): if 'logger' in self or 'logger' in self.dict(): self.logger.error(\ "None of requested datasets %s found in %s ..." \ %(str(names),filename) + \ " trying default MODIS names: only %s"\ %(str(contents))) names = def_names alt_names = def_alt_names for i in xrange(len(np.atleast_1d(contents))): if contents[i] in names: datasets.append(i) if not len(np.atleast_1d(datasets)): self.error_msg = "None of requested datasets %s found in %s"\ %(str(names),filename) + ' ' + \ "... trying default MODIS names: only %s"\ %(str(contents)) self.error = True if 'logger' in self or 'logger' in self.dict(): self.logger.error(self.error_msg) return 0, (self.error, self.error_msg) trans_names = {} for (i, j) in enumerate(alt_names): trans_names[names[i]] = j #trans_names = {names[i]:j for (i,j) in enumerate(alt_names)} alt_name = [] this_name = [] for i in datasets: this_name.append(contents[i]) alt_name.append(trans_names[contents[i]]) # Translate some old stylies... trans = {'raa': 'vaa', 'doys': 'time'} for i in trans: if i in contents: ncontents[np.where(contents == i)[0]] = trans[i] # as a minimum, there needs to be some definition of one of # the terms in location # check how many dimensions this has # now find a dataset try: # This could be more general, but this will do for now as its useful # for spatial datasets QA_OK = np.array(\ [8, 72, 136, 200, 1032, 1288, 2056,2120, 2184, 2248]) doy = f['doys'] - 2004000 qa = f['qa'] vza = f['vza'] sza = f['sza'] raa = f['raa'] y = [] for i in this_name: y.append(f[i]) #mask = np.logical_or.reduce([qa==x for x in QA_OK ]) if 'logger' in self or 'logger' in self.dict(): self.logger.info(\ "sucessfully interpreted NpzFile dataset from %s"\ %filename) self.logger.info("sub-setting ...") controls = [] locations = [] grid = [] qlocations = [] thisshape = vza.shape starter = {'time': np.min(doy), 'row': 0, 'col': 0} delta = {'time': 1, 'row': 1, 'col': 1} if len(np.atleast_1d(limits)) < 3: from eoldas_Lib import set_default_limits old_loc = location location = np.array(['time', 'row', 'col']) lim2 = set_default_limits(location) for i in xrange(len(np.atleast_1d(limits))): ww = np.where(old_loc[i] == location)[0] lim2[ww] = list(limits[i]) limits = lim2 for i in xrange(len(np.atleast_1d(limits))): if limits[i][0] == None: limits[i][0] = starter[location[i]] if limits[i][1] == None: limits[i][1] = (thisshape[i] - 1) + starter[location[i]] if limits[i][2] == None: limits[i][2] = delta[location[i]] limits = np.array(limits) start_doy = limits[0][0] end_doy = limits[0][1] step_doy = limits[0][2] start_row = limits[1][0] end_row = limits[1][1] step_row = limits[1][2] start_col = limits[2][0] end_col = limits[2][1] step_col = limits[2][2] gooddays = np.logical_and.reduce(np.concatenate(\ ([doy >= start_doy],[doy <=end_doy]))) qa = qa[gooddays, start_row:end_row + 1, start_col:end_col + 1] vza = vza[gooddays, start_row:end_row + 1, start_col:end_col + 1] * 0.01 sza = sza[gooddays, start_row:end_row + 1, start_col:end_col + 1] * 0.01 raa = raa[gooddays, start_row:end_row + 1, start_col:end_col + 1] * 0.01 yy = [] for i in xrange(len(np.atleast_1d(this_name))): this = y[i] yy.append(this[gooddays,start_row:end_row+1,\ start_col:end_col+1]*0.0001) doy = doy[gooddays] # now do QA mask = np.zeros_like(qa).astype(bool) # loop over qa for j in xrange(len(np.atleast_1d(QA_OK))): ww = np.where(qa == QA_OK[j]) mask[ww] = True # better look over data to check valid for j in xrange(len(np.atleast_1d(yy))): ww = np.where(yy[j] < 0) mask[ww] = False ww = np.where(mask) if 'logger' in self or 'logger' in self.dict(): self.logger.debug('parsing dataset: %d samples look ok'\ %np.array(ww).shape[1]) vza = vza[ww] sza = sza[ww] raa = raa[ww] doy = doy[ww[0]] row = ww[1] + start_row col = ww[2] + start_col locations = np.array([doy, row, col]) nnn = len(np.atleast_1d(locations[0])) orig = np.repeat(np.array([start_doy, start_row, start_col]), locations.shape[1]).reshape(locations.shape).T div = np.repeat(np.array([step_doy, step_row, step_col]), locations.shape[1]).reshape(locations.shape).T qlocations = ((locations.T - orig) / div.astype(float)).astype(int).T controls = np.array([np.ones_like(doy).astype(bool),\ vza,raa,sza,0*doy]) y = [] for i in xrange(len(np.atleast_1d(this_name))): this = yy[i] y.append(this[ww]) grid = np.array(y) fmt = 'BRDF-UCL' control = ['mask', 'vza', 'vaa', 'sza', 'saa'] bands = alt_name if not np.array(grid).size: if 'logger' in self or 'logger' in self.dict(): self.logger.error(\ "Warning: returning a zero-sized dataset ... "+\ " I wouldn;t try to do anything with it") # in case we dont have data for all bands mask = np.logical_or.reduce([[this_name[i]==x for x in names] \ for i in xrange(len(np.atleast_1d(this_name)))]) sd = np.array('0.004 0.015 0.003 0.004 0.013 0.01 0.006'\ .split())[mask] sd = np.array([float(i) for i in sd.flatten()])\ .reshape(sd.shape) nsamps = grid.shape[1] sd = sd.repeat(nsamps).reshape(grid.shape).T datasets = ParamStorage() datasets.data = ParamStorage() datasets.name = ParamStorage() datasets.name.fmt = fmt grid = grid.T datasets.data[name] = np.zeros([grid.shape[0],len(np.atleast_1d(names))])\ .astype(object) datasets.data[name][:, :] = None for i in xrange(len(np.atleast_1d(this_name))): ww = np.where(names == this_name[i])[0][0] datasets.data[name][:, ww] = grid[:, i] datasets.data.location = np.array(locations).T datasets.data.control = np.array(controls).T datasets.data.qlocation = np.array(qlocations).T datasets.name[name] = np.array(names) datasets.name.location = np.array(['time', 'row', 'col']) datasets.name.control = np.array(control) datasets.name.qlocation = limits datasets.name.bands = np.array(bands) datasets.data.sd = np.zeros([grid.shape[0],len(np.atleast_1d(names))])\ .astype(object) # for i in xrange(grid.shape[0]): # datasets.data.sd[i,:] = self.options.sd datasets.data.sd[:, :] = None for i in xrange(len(np.atleast_1d(this_name))): ww = np.where(names == this_name[i])[0][0] datasets.data.sd[:, ww] = sd[:, i] datasets.name.sd = np.array(names) if 'logger' in self or 'logger' in self.dict(): self.logger.debug('finished parsing dataset') except: self.error_msg=\ "a problem processing information from %s as a NpzFile"\ %filename self.error = True if 'logger' in self or 'logger' in self.dict(): self.logger.info(self.error_msg) return 0, (self.error, self.error_msg) f.close() if 'logger' in self or 'logger' in self.dict(): self.logger.info('... done') self.error = False self.error_msg = "" return datasets, (self.error, self.error_msg)
def demonstration(): from eoldas_State import State from eoldas_ParamStorage import ParamStorage import numpy as np # a basic set up for State, setting names & bounds etc options = ParamStorage() options.logfile = 'test/data_type/logs/log.dat' options.names = \ 'gamma xlai xhc rpl xkab scen xkw xkm xleafn xs1 xs2 xs3 xs4 lad'.split() options.bounds = [[0.01,None],\ [0.01,0.99],\ [0.01,10.0],\ [0.001,0.10],\ [0.1,0.99],\ [0.0,1.0],\ [0.01,0.99],\ [0.3,0.9],\ [0.9,2.5],\ [0.0, 4.],\ [0.0, 5.],\ [None, None],\ [None, None],\ [None, None]] options.default = -1.0*np.ones(len(options.names)) options.location = 'time'.split() options.control = 'mask vza vaa sza saa'.split() options.datadir = ['.','test/data_type'] name = "eoldas_data_type test 0" options.limits = [[170,365,1]] self = State(options,datatype='y',name=name,datadir=\ options.datadir,env=None,logfile=options.logfile) self.tester() # Now we set some state data this = ParamStorage() # how many state vector elements should there be? n_states = len(self.Name.state) self.state = np.ones([100,n_states]) self.Data.sd = np.ones([1,n_states]) self.Name.sd = self.Name.state print '******************' print (self.Data.sd,self.Name.sd) this.data = ParamStorage() this.name = ParamStorage() this.data.state = self.state *2 controls = self.Name.control n_controls = len(controls) this.data.control = np.ones([100,n_controls]) this.data.location = np.ones([100,n_controls]) # we can load x_state from a ParamStorage self.state = this # now we should see the control data etc. self.tester() # change a dataset name to see if that works: # should load everything as numpy arrays self.Name.control = np.array(['vza','sza']) # change a dataset to see if that works .. deliberately load a bad one self.Data.control = 0 # now try a direct state data load self.state = np.zeros([100,100])+5. # which will load into self.Data.state self['state'] = np.zeros([100,100])+6. #now try accessing it: print self.state # change the control info self.Name.control = np.array(['vza']) print self.Name.control # reset it self.state = self.state * 2 print self.state # now try reading a file into state self.state = 'test/data_type/input/test.brf' print '========' print 'data from a BRDF file' self.tester() print '========' # written as a pickle self.write('test/data_type/output/test.pickle',None,fmt='pickle') self.logger.info("...DONE...") name = "eoldas_data_type test 1" del self self1 = State(options,datatype='y',name=name,datadir=\ options.datadir,env=None,logfile=options.logfile,grid=True) # read from pickle self1.state = 'test/data_type/output/test.pickle' print '========' print 'data from a pickle file' self1.tester() print '========' # try to load an npx file del self1 options.location = 'time row col'.split() options.limits = [[170,365,1],[0,500,1],[200,200,1]] self2 = State(options,datatype='y',name=name,datadir=\ options.datadir,env=None,logfile=options.logfile) self2.state = 'test/data_type/input/interpolated_data.npz' print '========' print 'data from a npz file' self2.tester() print '========' # write as BRDF-UCL self2.write('test/data_type/output/test.brf',None,fmt='BRDF-UCL') del self2 self3 = State(options,datatype='y',name=name,datadir=\ options.datadir,env=None,logfile=options.logfile) # then test the reader self3.state = 0. self3.state = 'test/data_type/output/test.brf' print '========' print 'data from a BRDF-UCL file' print '========' self3.tester() print '========' # then write as a PARAMETERS file self3.write('test/data_type/output/test.param',None,fmt='PARAMETERS') del self3 options.location = 'time row col'.split() options.limits = [[170,365,1],[0,500,1],[200,200,1]] options.control = np.array(['mask','vza','vaa','sza','saa']) self4 = State(options,datatype='y',name=name,datadir=\ options.datadir,env=None,logfile=options.logfile) # then test the reader self4.state = 0. self4.state = 'test/data_type/output/test.param' print '========' print 'data from a PARAMETERS file' print '========' self4.tester() print '========'
def read_numpy(self,filename,name,info=[]): ''' Try to read the file as as a NpzFile file ''' from eoldas_Lib import set_default_limits,check_limits_valid,\ quantize_location,dequantize_location # none of these ciritical to functioning try: info = self._state.info except: info = [] try: names = self.name.state except: try: names = self.Name.state except: names = None try: control = self.Name.control except: try: control = self.name.control except: control = None try: location = self.name.location except: try: location = self.Name.location except: location = ['time','row','col'] try: limits = self.name.qlocation except: try: limits = self.Name.qlocation except: limits = set_default_limits(location) # refl_check=False,names=None,\ # control=['mask','vza','vaa','sza','saa'],\ # location=['time','row','col'],limits=None # location specifies the dimesions and names of the # problem, e.g., & typically [time,row,col] limits = np.array(check_limits_valid(limits)) try: f = np.load(filename) if not type(f).__name__ == 'NpzFile': f.close() self.error_msg="%s is not a NpzFile"%filename self.error=True if 'logger' in self or 'logger' in self.dict(): self.logger.info(self.error_msg) return 0,(self.error,self.error_msg) except: self.error_msg="a problem opening %s as a NpzFile"%filename self.error=True if 'logger' in self or 'logger' in self.dict(): self.logger.info(self.error_msg) return 0,(self.error,self.error_msg) # ok so far then # lets have a look inside ncontents = np.array(f.files) contents = np.array(f.files) # translation table for default names def_names = 'b1 b2 b3 b4 b5 b6 b7'.split() if names == None: # assume MODIS names = def_names def_alt_names = \ '645.5 856.5 465.6 553.6 1241.6 1629.1 2114.1'.split() # look for any of names in contents datasets = [] alt_datasets = [] alt_names = names for i in xrange(len(np.atleast_1d(contents))): if contents[i] in names: datasets.append(i) if not len(np.atleast_1d(datasets)): if 'logger' in self or 'logger' in self.dict(): self.logger.error(\ "None of requested datasets %s found in %s ..." \ %(str(names),filename) + \ " trying default MODIS names: only %s"\ %(str(contents))) names = def_names alt_names = def_alt_names for i in xrange(len(np.atleast_1d(contents))): if contents[i] in names: datasets.append(i) if not len(np.atleast_1d(datasets)): self.error_msg = "None of requested datasets %s found in %s"\ %(str(names),filename) + ' ' + \ "... trying default MODIS names: only %s"\ %(str(contents)) self.error = True if 'logger' in self or 'logger' in self.dict(): self.logger.error(self.error_msg) return 0,(self.error,self.error_msg) trans_names = {} for (i,j) in enumerate(alt_names): trans_names[names[i]] = j #trans_names = {names[i]:j for (i,j) in enumerate(alt_names)} alt_name = [] this_name = [] for i in datasets: this_name.append(contents[i]) alt_name.append(trans_names[contents[i]]) # Translate some old stylies... trans = {'raa':'vaa','doys':'time'} for i in trans: if i in contents: ncontents[np.where(contents==i)[0]]=trans[i] # as a minimum, there needs to be some definition of one of # the terms in location # check how many dimensions this has # now find a dataset try: # This could be more general, but this will do for now as its useful # for spatial datasets QA_OK = np.array(\ [8, 72, 136, 200, 1032, 1288, 2056,2120, 2184, 2248]) doy = f['doys'] - 2004000 qa = f['qa'] vza = f['vza'] sza = f['sza'] raa = f['raa'] y = [] for i in this_name: y.append(f[i]) #mask = np.logical_or.reduce([qa==x for x in QA_OK ]) if 'logger' in self or 'logger' in self.dict(): self.logger.info(\ "sucessfully interpreted NpzFile dataset from %s"\ %filename) self.logger.info("sub-setting ...") controls = [] locations = [] grid = [] qlocations = [] thisshape = vza.shape starter = {'time':np.min(doy),'row':0,'col':0} delta = {'time':1,'row':1,'col':1} if len(np.atleast_1d(limits)) <3: from eoldas_Lib import set_default_limits old_loc = location location = np.array(['time','row','col']) lim2 = set_default_limits(location) for i in xrange(len(np.atleast_1d(limits))): ww = np.where(old_loc[i] == location)[0] lim2[ww] = list(limits[i]) limits = lim2 for i in xrange(len(np.atleast_1d(limits))): if limits[i][0] == None: limits[i][0] = starter[location[i]] if limits[i][1] == None: limits[i][1] = (thisshape[i]-1) + starter[location[i]] if limits[i][2] == None: limits[i][2]= delta[location[i]] limits = np.array(limits) start_doy = limits[0][0] end_doy = limits[0][1] step_doy = limits[0][2] start_row = limits[1][0] end_row = limits[1][1] step_row = limits[1][2] start_col = limits[2][0] end_col = limits[2][1] step_col = limits[2][2] gooddays = np.logical_and.reduce(np.concatenate(\ ([doy >= start_doy],[doy <=end_doy]))) qa = qa[gooddays,start_row:end_row+1,start_col:end_col+1] vza = vza[gooddays,start_row:end_row+1,start_col:end_col+1]*0.01 sza = sza[gooddays,start_row:end_row+1,start_col:end_col+1]*0.01 raa = raa[gooddays,start_row:end_row+1,start_col:end_col+1]*0.01 yy = [] for i in xrange(len(np.atleast_1d(this_name))): this = y[i] yy.append(this[gooddays,start_row:end_row+1,\ start_col:end_col+1]*0.0001) doy = doy[gooddays] # now do QA mask = np.zeros_like(qa).astype(bool) # loop over qa for j in xrange(len(np.atleast_1d(QA_OK))): ww = np.where(qa==QA_OK[j]) mask[ww] = True # better look over data to check valid for j in xrange(len(np.atleast_1d(yy))): ww = np.where(yy[j] < 0) mask[ww] = False ww = np.where(mask) if 'logger' in self or 'logger' in self.dict(): self.logger.debug('parsing dataset: %d samples look ok'\ %np.array(ww).shape[1]) vza = vza[ww] sza = sza[ww] raa = raa[ww] doy= doy[ww[0]] row = ww[1]+start_row col = ww[2]+start_col locations = np.array([doy,row,col]) nnn = len(np.atleast_1d(locations[0])) orig = np.repeat(np.array([start_doy,start_row,start_col]),locations.shape[1]).reshape(locations.shape).T div = np.repeat(np.array([step_doy,step_row,step_col]),locations.shape[1]).reshape(locations.shape).T qlocations = ((locations.T - orig)/div.astype(float)).astype(int).T controls = np.array([np.ones_like(doy).astype(bool),\ vza,raa,sza,0*doy]) y = [] for i in xrange(len(np.atleast_1d(this_name))): this = yy[i] y.append(this[ww]) grid = np.array(y) fmt = 'BRDF-UCL' control = ['mask','vza','vaa','sza','saa'] bands = alt_name if not np.array(grid).size: if 'logger' in self or 'logger' in self.dict(): self.logger.error(\ "Warning: returning a zero-sized dataset ... "+\ " I wouldn;t try to do anything with it") # in case we dont have data for all bands mask = np.logical_or.reduce([[this_name[i]==x for x in names] \ for i in xrange(len(np.atleast_1d(this_name)))]) sd = np.array('0.004 0.015 0.003 0.004 0.013 0.01 0.006'\ .split())[mask] sd = np.array([float(i) for i in sd.flatten()])\ .reshape(sd.shape) nsamps = grid.shape[1] sd = sd.repeat(nsamps).reshape(grid.shape).T datasets = ParamStorage() datasets.data = ParamStorage() datasets.name = ParamStorage() datasets.name.fmt = fmt grid = grid.T datasets.data[name] = np.zeros([grid.shape[0],len(np.atleast_1d(names))])\ .astype(object) datasets.data[name][:,:] = None for i in xrange(len(np.atleast_1d(this_name))): ww = np.where(names == this_name[i])[0][0] datasets.data[name][:,ww] = grid[:,i] datasets.data.location = np.array(locations).T datasets.data.control = np.array(controls).T datasets.data.qlocation = np.array(qlocations).T datasets.name[name] = np.array(names) datasets.name.location = np.array(['time','row','col']) datasets.name.control = np.array(control) datasets.name.qlocation = limits datasets.name.bands = np.array(bands) datasets.data.sd = np.zeros([grid.shape[0],len(np.atleast_1d(names))])\ .astype(object) # for i in xrange(grid.shape[0]): # datasets.data.sd[i,:] = self.options.sd datasets.data.sd[:,:] = None for i in xrange(len(np.atleast_1d(this_name))): ww = np.where(names == this_name[i])[0][0] datasets.data.sd[:,ww] = sd[:,i] datasets.name.sd = np.array(names) if 'logger' in self or 'logger' in self.dict(): self.logger.debug('finished parsing dataset') except: self.error_msg=\ "a problem processing information from %s as a NpzFile"\ %filename self.error=True if 'logger' in self or 'logger' in self.dict(): self.logger.info(self.error_msg) return 0,(self.error,self.error_msg) f.close() if 'logger' in self or 'logger' in self.dict(): self.logger.info('... done') self.error=False self.error_msg="" return datasets,(self.error,self.error_msg)
def demonstration(): from eoldas_State import State from eoldas_ParamStorage import ParamStorage import numpy as np # a basic set up for State, setting names & bounds etc options = ParamStorage() options.logfile = 'test/data_type/logs/log.dat' options.names = \ 'gamma xlai xhc rpl xkab scen xkw xkm xleafn xs1 xs2 xs3 xs4 lad'.split() options.bounds = [[0.01,None],\ [0.01,0.99],\ [0.01,10.0],\ [0.001,0.10],\ [0.1,0.99],\ [0.0,1.0],\ [0.01,0.99],\ [0.3,0.9],\ [0.9,2.5],\ [0.0, 4.],\ [0.0, 5.],\ [None, None],\ [None, None],\ [None, None]] options.default = -1.0 * np.ones(len(options.names)) options.location = 'time'.split() options.control = 'mask vza vaa sza saa'.split() options.datadir = ['.', 'test/data_type'] name = "eoldas_data_type test 0" options.limits = [[170, 365, 1]] self = State(options,datatype='y',name=name,datadir=\ options.datadir,env=None,logfile=options.logfile) self.tester() # Now we set some state data this = ParamStorage() # how many state vector elements should there be? n_states = len(self.Name.state) self.state = np.ones([100, n_states]) self.Data.sd = np.ones([1, n_states]) self.Name.sd = self.Name.state print '******************' print(self.Data.sd, self.Name.sd) this.data = ParamStorage() this.name = ParamStorage() this.data.state = self.state * 2 controls = self.Name.control n_controls = len(controls) this.data.control = np.ones([100, n_controls]) this.data.location = np.ones([100, n_controls]) # we can load x_state from a ParamStorage self.state = this # now we should see the control data etc. self.tester() # change a dataset name to see if that works: # should load everything as numpy arrays self.Name.control = np.array(['vza', 'sza']) # change a dataset to see if that works .. deliberately load a bad one self.Data.control = 0 # now try a direct state data load self.state = np.zeros([100, 100]) + 5. # which will load into self.Data.state self['state'] = np.zeros([100, 100]) + 6. #now try accessing it: print self.state # change the control info self.Name.control = np.array(['vza']) print self.Name.control # reset it self.state = self.state * 2 print self.state # now try reading a file into state self.state = 'test/data_type/input/test.brf' print '========' print 'data from a BRDF file' self.tester() print '========' # written as a pickle self.write('test/data_type/output/test.pickle', None, fmt='pickle') self.logger.info("...DONE...") name = "eoldas_data_type test 1" del self self1 = State(options,datatype='y',name=name,datadir=\ options.datadir,env=None,logfile=options.logfile,grid=True) # read from pickle self1.state = 'test/data_type/output/test.pickle' print '========' print 'data from a pickle file' self1.tester() print '========' # try to load an npx file del self1 options.location = 'time row col'.split() options.limits = [[170, 365, 1], [0, 500, 1], [200, 200, 1]] self2 = State(options,datatype='y',name=name,datadir=\ options.datadir,env=None,logfile=options.logfile) self2.state = 'test/data_type/input/interpolated_data.npz' print '========' print 'data from a npz file' self2.tester() print '========' # write as BRDF-UCL self2.write('test/data_type/output/test.brf', None, fmt='BRDF-UCL') del self2 self3 = State(options,datatype='y',name=name,datadir=\ options.datadir,env=None,logfile=options.logfile) # then test the reader self3.state = 0. self3.state = 'test/data_type/output/test.brf' print '========' print 'data from a BRDF-UCL file' print '========' self3.tester() print '========' # then write as a PARAMETERS file self3.write('test/data_type/output/test.param', None, fmt='PARAMETERS') del self3 options.location = 'time row col'.split() options.limits = [[170, 365, 1], [0, 500, 1], [200, 200, 1]] options.control = np.array(['mask', 'vza', 'vaa', 'sza', 'saa']) self4 = State(options,datatype='y',name=name,datadir=\ options.datadir,env=None,logfile=options.logfile) # then test the reader self4.state = 0. self4.state = 'test/data_type/output/test.param' print '========' print 'data from a PARAMETERS file' print '========' self4.tester() print '========'