def read_numpy_fromfile(self, goodfile, dataset, info=None):
'''
Utility to try to read a file goodfile
This simple version of the utility just tries a numpy.fromfile
It is not generally recommended to use this first
as it will skip headers etc that may contain
information
However, this is a good illustration of the required interface format
for file readers.
Inputs:
goodfile : a filename (ideally one that exists)
dataset : dataset name (e.g. x_state)
info : a list of other information
Outputs:
retval : ParamStorage
error : tuple
where:
retval contains dataset 'this' in data['this']
and other information (e.g. locations etc) in
retval.data and retval.names
'''
#self.store_header = goodfile.open().readline().close()
this = np.genfromtxt(goodfile, skip_header=1)
l = len(np.atleast_1d(this).shape)
if this.size > 0:
retval = ParamStorage()
retval.name = ParamStorage()
retval.data = ParamStorage()
retval.data[dataset] = this
retval.name.filename = goodfile
retval.name.fmt = 'np.genfromtxt'
return retval, (False, "")
error = True
error_msg = "Failed numpy read of %s" % goodfile
return 0, (error, error_msg)
def read_numpy_fromfile(self,goodfile,dataset,info=None):
'''
Utility to try to read a file goodfile
This simple version of the utility just tries a numpy.fromfile
It is not generally recommended to use this first
as it will skip headers etc that may contain
information
However, this is a good illustration of the required interface format
for file readers.
Inputs:
goodfile : a filename (ideally one that exists)
dataset : dataset name (e.g. x_state)
info : a list of other information
Outputs:
retval : ParamStorage
error : tuple
where:
retval contains dataset 'this' in data['this']
and other information (e.g. locations etc) in
retval.data and retval.names
'''
#self.store_header = goodfile.open().readline().close()
this = np.genfromtxt(goodfile,skip_header=1)
l = len(np.atleast_1d(this).shape)
if this.size > 0:
retval = ParamStorage()
retval.name = ParamStorage()
retval.data = ParamStorage()
retval.data[dataset] = this
retval.name.filename = goodfile
retval.name.fmt = 'np.genfromtxt'
return retval,(False,"")
error = True
error_msg = "Failed numpy read of %s"%goodfile
return 0,(error,error_msg)
def demonstration():
# set state to a filename
# and it will be loaded with the data
x = DemoClass()
data = {'state':np.ones(2)*5. ,'foo':np.ones(10)}
name = {'state':'of the nation','foo':'bar'}
this = {'data':data,'name':name}
x.state = this
print 1,x.state,x['state']
x.oats = 'beans and barley-o'
# nothing set so far
print 2,x['state']
# should return the same
print 3,x.state
x.state = 'test/data_type/input/test.brf'
print 4,x.state
print 5,x.Name.fmt
# set state to a dict and
# it will load from that
data = {'state':np.zeros(10)}
name = {'state':'foo'}
x.state = {'data':data,'name':name}
print 6,x.state
# set from a ParamStorage
# and it will be loaded
this = ParamStorage()
this.data = ParamStorage()
this.name = ParamStorage()
this.data.state = np.ones(10)
this.name.state = 'bar'
this.data.sd = np.ones(10)*2.
this.name.sd = 'sd info'
# assign the data
x.state = this
# access the data
print 7,x.state
# access another member
# Data, Name == implicitly .state
print 8,x.Data.sd
print 9,x.Name.sd
# set directly
x.Name.sd = 'bar'
print 10,x.Name.sd
# set from a tuple (data,name)
# or a list [data,name]
data = 'foo'
name = 'bar'
x.state = (data,name)
print 11,x.state
x.state = [name,data]
print 12,x.state
# set from a numpy array
x.state = np.array(np.arange(10))
print 13,x.state
# set from another state
y = DemoClass()
y.state = x.state
x.state = x.state * 2
print 'x state',x.state
print 'y state',y.state
# set from a float
x.state = 100.
print 14,x.state
# another interesting feature
# we have 2 special terms in demonstration
# state and other
# if we set up some strcture for data
# for other
this = ParamStorage()
this.data = ParamStorage()
this.name = ParamStorage()
this.data.other = np.ones(10)
this.name.other = 'bar'
# and the assign it to state
x.state = this
print 15,'state',x.state
# we see state is unchanged
# but other is also not set.
print 16,'other',x.other
# we load into other using:
x.other = this
print 'other',x.other
# but if you look at the information contained
print 17,x._other.to_dict()
print 18,x._state.to_dict()
# or better writtem as:
print 19,x.var('state').to_dict()
# you will see that state contains the other data that was loaded
# a simple way to write out the data is to a pickle
# x.write_pickle('xstate','x_state.pkl')
# but try to avoid using the underscores
print 20,"x state in pickle:",x.state
SpecialVariable.write(x._state,'x_state.pkl',fmt='pickle')
# which we can reload:
z = DemoClass()
z.state = 'x_state.pkl'
print 21,"z state read from pickle",z.state
# which is the same as a forced read ...
zz = DemoClass()
zz.state = 'x_state.pkl'
print 22,zz.state
# read a brf file
zz.Name.qlocation = [[170,365,1],[0,500,1],[200,200,1]]
zz.state = 'test/data_type/input/interpolated_data.npz'
print zz.state
SpecialVariable.write(zz._state,'test/data_type/output/interpolated_data.pkl',fmt='pickle')
zz.state = 'test/data_type/input/test.brf'
print zz.state
def read_numpy(self, filename, name, info=[]):
'''
Try to read the file as as a NpzFile file
'''
from eoldas_Lib import set_default_limits,check_limits_valid,\
quantize_location,dequantize_location
# none of these ciritical to functioning
try:
info = self._state.info
except:
info = []
try:
names = self.name.state
except:
try:
names = self.Name.state
except:
names = None
try:
control = self.Name.control
except:
try:
control = self.name.control
except:
control = None
try:
location = self.name.location
except:
try:
location = self.Name.location
except:
location = ['time', 'row', 'col']
try:
limits = self.name.qlocation
except:
try:
limits = self.Name.qlocation
except:
limits = set_default_limits(location)
# refl_check=False,names=None,\
# control=['mask','vza','vaa','sza','saa'],\
# location=['time','row','col'],limits=None
# location specifies the dimesions and names of the
# problem, e.g., & typically [time,row,col]
limits = np.array(check_limits_valid(limits))
try:
f = np.load(filename)
if not type(f).__name__ == 'NpzFile':
f.close()
self.error_msg = "%s is not a NpzFile" % filename
self.error = True
if 'logger' in self or 'logger' in self.dict():
self.logger.info(self.error_msg)
return 0, (self.error, self.error_msg)
except:
self.error_msg = "a problem opening %s as a NpzFile" % filename
self.error = True
if 'logger' in self or 'logger' in self.dict():
self.logger.info(self.error_msg)
return 0, (self.error, self.error_msg)
# ok so far then
# lets have a look inside
ncontents = np.array(f.files)
contents = np.array(f.files)
# translation table for default names
def_names = 'b1 b2 b3 b4 b5 b6 b7'.split()
if names == None:
# assume MODIS
names = def_names
def_alt_names = \
'645.5 856.5 465.6 553.6 1241.6 1629.1 2114.1'.split()
# look for any of names in contents
datasets = []
alt_datasets = []
alt_names = names
for i in xrange(len(np.atleast_1d(contents))):
if contents[i] in names:
datasets.append(i)
if not len(np.atleast_1d(datasets)):
if 'logger' in self or 'logger' in self.dict():
self.logger.error(\
"None of requested datasets %s found in %s ..." \
%(str(names),filename) + \
" trying default MODIS names: only %s"\
%(str(contents)))
names = def_names
alt_names = def_alt_names
for i in xrange(len(np.atleast_1d(contents))):
if contents[i] in names:
datasets.append(i)
if not len(np.atleast_1d(datasets)):
self.error_msg = "None of requested datasets %s found in %s"\
%(str(names),filename) + ' ' + \
"... trying default MODIS names: only %s"\
%(str(contents))
self.error = True
if 'logger' in self or 'logger' in self.dict():
self.logger.error(self.error_msg)
return 0, (self.error, self.error_msg)
trans_names = {}
for (i, j) in enumerate(alt_names):
trans_names[names[i]] = j
#trans_names = {names[i]:j for (i,j) in enumerate(alt_names)}
alt_name = []
this_name = []
for i in datasets:
this_name.append(contents[i])
alt_name.append(trans_names[contents[i]])
# Translate some old stylies...
trans = {'raa': 'vaa', 'doys': 'time'}
for i in trans:
if i in contents:
ncontents[np.where(contents == i)[0]] = trans[i]
# as a minimum, there needs to be some definition of one of
# the terms in location
# check how many dimensions this has
# now find a dataset
try:
# This could be more general, but this will do for now as its useful
# for spatial datasets
QA_OK = np.array(\
[8, 72, 136, 200, 1032, 1288, 2056,2120, 2184, 2248])
doy = f['doys'] - 2004000
qa = f['qa']
vza = f['vza']
sza = f['sza']
raa = f['raa']
y = []
for i in this_name:
y.append(f[i])
#mask = np.logical_or.reduce([qa==x for x in QA_OK ])
if 'logger' in self or 'logger' in self.dict():
self.logger.info(\
"sucessfully interpreted NpzFile dataset from %s"\
%filename)
self.logger.info("sub-setting ...")
controls = []
locations = []
grid = []
qlocations = []
thisshape = vza.shape
starter = {'time': np.min(doy), 'row': 0, 'col': 0}
delta = {'time': 1, 'row': 1, 'col': 1}
if len(np.atleast_1d(limits)) < 3:
from eoldas_Lib import set_default_limits
old_loc = location
location = np.array(['time', 'row', 'col'])
lim2 = set_default_limits(location)
for i in xrange(len(np.atleast_1d(limits))):
ww = np.where(old_loc[i] == location)[0]
lim2[ww] = list(limits[i])
limits = lim2
for i in xrange(len(np.atleast_1d(limits))):
if limits[i][0] == None:
limits[i][0] = starter[location[i]]
if limits[i][1] == None:
limits[i][1] = (thisshape[i] - 1) + starter[location[i]]
if limits[i][2] == None:
limits[i][2] = delta[location[i]]
limits = np.array(limits)
start_doy = limits[0][0]
end_doy = limits[0][1]
step_doy = limits[0][2]
start_row = limits[1][0]
end_row = limits[1][1]
step_row = limits[1][2]
start_col = limits[2][0]
end_col = limits[2][1]
step_col = limits[2][2]
gooddays = np.logical_and.reduce(np.concatenate(\
([doy >= start_doy],[doy <=end_doy])))
qa = qa[gooddays, start_row:end_row + 1, start_col:end_col + 1]
vza = vza[gooddays, start_row:end_row + 1,
start_col:end_col + 1] * 0.01
sza = sza[gooddays, start_row:end_row + 1,
start_col:end_col + 1] * 0.01
raa = raa[gooddays, start_row:end_row + 1,
start_col:end_col + 1] * 0.01
yy = []
for i in xrange(len(np.atleast_1d(this_name))):
this = y[i]
yy.append(this[gooddays,start_row:end_row+1,\
start_col:end_col+1]*0.0001)
doy = doy[gooddays]
# now do QA
mask = np.zeros_like(qa).astype(bool)
# loop over qa
for j in xrange(len(np.atleast_1d(QA_OK))):
ww = np.where(qa == QA_OK[j])
mask[ww] = True
# better look over data to check valid
for j in xrange(len(np.atleast_1d(yy))):
ww = np.where(yy[j] < 0)
mask[ww] = False
ww = np.where(mask)
if 'logger' in self or 'logger' in self.dict():
self.logger.debug('parsing dataset: %d samples look ok'\
%np.array(ww).shape[1])
vza = vza[ww]
sza = sza[ww]
raa = raa[ww]
doy = doy[ww[0]]
row = ww[1] + start_row
col = ww[2] + start_col
locations = np.array([doy, row, col])
nnn = len(np.atleast_1d(locations[0]))
orig = np.repeat(np.array([start_doy, start_row, start_col]),
locations.shape[1]).reshape(locations.shape).T
div = np.repeat(np.array([step_doy, step_row, step_col]),
locations.shape[1]).reshape(locations.shape).T
qlocations = ((locations.T - orig) / div.astype(float)).astype(int).T
controls = np.array([np.ones_like(doy).astype(bool),\
vza,raa,sza,0*doy])
y = []
for i in xrange(len(np.atleast_1d(this_name))):
this = yy[i]
y.append(this[ww])
grid = np.array(y)
fmt = 'BRDF-UCL'
control = ['mask', 'vza', 'vaa', 'sza', 'saa']
bands = alt_name
if not np.array(grid).size:
if 'logger' in self or 'logger' in self.dict():
self.logger.error(\
"Warning: returning a zero-sized dataset ... "+\
" I wouldn;t try to do anything with it")
# in case we dont have data for all bands
mask = np.logical_or.reduce([[this_name[i]==x for x in names] \
for i in xrange(len(np.atleast_1d(this_name)))])
sd = np.array('0.004 0.015 0.003 0.004 0.013 0.01 0.006'\
.split())[mask]
sd = np.array([float(i) for i in sd.flatten()])\
.reshape(sd.shape)
nsamps = grid.shape[1]
sd = sd.repeat(nsamps).reshape(grid.shape).T
datasets = ParamStorage()
datasets.data = ParamStorage()
datasets.name = ParamStorage()
datasets.name.fmt = fmt
grid = grid.T
datasets.data[name] = np.zeros([grid.shape[0],len(np.atleast_1d(names))])\
.astype(object)
datasets.data[name][:, :] = None
for i in xrange(len(np.atleast_1d(this_name))):
ww = np.where(names == this_name[i])[0][0]
datasets.data[name][:, ww] = grid[:, i]
datasets.data.location = np.array(locations).T
datasets.data.control = np.array(controls).T
datasets.data.qlocation = np.array(qlocations).T
datasets.name[name] = np.array(names)
datasets.name.location = np.array(['time', 'row', 'col'])
datasets.name.control = np.array(control)
datasets.name.qlocation = limits
datasets.name.bands = np.array(bands)
datasets.data.sd = np.zeros([grid.shape[0],len(np.atleast_1d(names))])\
.astype(object)
# for i in xrange(grid.shape[0]):
# datasets.data.sd[i,:] = self.options.sd
datasets.data.sd[:, :] = None
for i in xrange(len(np.atleast_1d(this_name))):
ww = np.where(names == this_name[i])[0][0]
datasets.data.sd[:, ww] = sd[:, i]
datasets.name.sd = np.array(names)
if 'logger' in self or 'logger' in self.dict():
self.logger.debug('finished parsing dataset')
except:
self.error_msg=\
"a problem processing information from %s as a NpzFile"\
%filename
self.error = True
if 'logger' in self or 'logger' in self.dict():
self.logger.info(self.error_msg)
return 0, (self.error, self.error_msg)
f.close()
if 'logger' in self or 'logger' in self.dict():
self.logger.info('... done')
self.error = False
self.error_msg = ""
return datasets, (self.error, self.error_msg)
def demonstration():
from eoldas_State import State
from eoldas_ParamStorage import ParamStorage
import numpy as np
# a basic set up for State, setting names & bounds etc
options = ParamStorage()
options.logfile = 'test/data_type/logs/log.dat'
options.names = \
'gamma xlai xhc rpl xkab scen xkw xkm xleafn xs1 xs2 xs3 xs4 lad'.split()
options.bounds = [[0.01,None],\
[0.01,0.99],\
[0.01,10.0],\
[0.001,0.10],\
[0.1,0.99],\
[0.0,1.0],\
[0.01,0.99],\
[0.3,0.9],\
[0.9,2.5],\
[0.0, 4.],\
[0.0, 5.],\
[None, None],\
[None, None],\
[None, None]]
options.default = -1.0*np.ones(len(options.names))
options.location = 'time'.split()
options.control = 'mask vza vaa sza saa'.split()
options.datadir = ['.','test/data_type']
name = "eoldas_data_type test 0"
options.limits = [[170,365,1]]
self = State(options,datatype='y',name=name,datadir=\
options.datadir,env=None,logfile=options.logfile)
self.tester()
# Now we set some state data
this = ParamStorage()
# how many state vector elements should there be?
n_states = len(self.Name.state)
self.state = np.ones([100,n_states])
self.Data.sd = np.ones([1,n_states])
self.Name.sd = self.Name.state
print '******************'
print (self.Data.sd,self.Name.sd)
this.data = ParamStorage()
this.name = ParamStorage()
this.data.state = self.state *2
controls = self.Name.control
n_controls = len(controls)
this.data.control = np.ones([100,n_controls])
this.data.location = np.ones([100,n_controls])
# we can load x_state from a ParamStorage
self.state = this
# now we should see the control data etc.
self.tester()
# change a dataset name to see if that works:
# should load everything as numpy arrays
self.Name.control = np.array(['vza','sza'])
# change a dataset to see if that works .. deliberately load a bad one
self.Data.control = 0
# now try a direct state data load
self.state = np.zeros([100,100])+5.
# which will load into self.Data.state
self['state'] = np.zeros([100,100])+6.
#now try accessing it:
print self.state
# change the control info
self.Name.control = np.array(['vza'])
print self.Name.control
# reset it
self.state = self.state * 2
print self.state
# now try reading a file into state
self.state = 'test/data_type/input/test.brf'
print '========'
print 'data from a BRDF file'
self.tester()
print '========'
# written as a pickle
self.write('test/data_type/output/test.pickle',None,fmt='pickle')
self.logger.info("...DONE...")
name = "eoldas_data_type test 1"
del self
self1 = State(options,datatype='y',name=name,datadir=\
options.datadir,env=None,logfile=options.logfile,grid=True)
# read from pickle
self1.state = 'test/data_type/output/test.pickle'
print '========'
print 'data from a pickle file'
self1.tester()
print '========'
# try to load an npx file
del self1
options.location = 'time row col'.split()
options.limits = [[170,365,1],[0,500,1],[200,200,1]]
self2 = State(options,datatype='y',name=name,datadir=\
options.datadir,env=None,logfile=options.logfile)
self2.state = 'test/data_type/input/interpolated_data.npz'
print '========'
print 'data from a npz file'
self2.tester()
print '========'
# write as BRDF-UCL
self2.write('test/data_type/output/test.brf',None,fmt='BRDF-UCL')
del self2
self3 = State(options,datatype='y',name=name,datadir=\
options.datadir,env=None,logfile=options.logfile)
# then test the reader
self3.state = 0.
self3.state = 'test/data_type/output/test.brf'
print '========'
print 'data from a BRDF-UCL file'
print '========'
self3.tester()
print '========'
# then write as a PARAMETERS file
self3.write('test/data_type/output/test.param',None,fmt='PARAMETERS')
del self3
options.location = 'time row col'.split()
options.limits = [[170,365,1],[0,500,1],[200,200,1]]
options.control = np.array(['mask','vza','vaa','sza','saa'])
self4 = State(options,datatype='y',name=name,datadir=\
options.datadir,env=None,logfile=options.logfile)
# then test the reader
self4.state = 0.
self4.state = 'test/data_type/output/test.param'
print '========'
print 'data from a PARAMETERS file'
print '========'
self4.tester()
print '========'
def read_numpy(self,filename,name,info=[]):
'''
Try to read the file as as a NpzFile file
'''
from eoldas_Lib import set_default_limits,check_limits_valid,\
quantize_location,dequantize_location
# none of these ciritical to functioning
try:
info = self._state.info
except:
info = []
try:
names = self.name.state
except:
try:
names = self.Name.state
except:
names = None
try:
control = self.Name.control
except:
try:
control = self.name.control
except:
control = None
try:
location = self.name.location
except:
try:
location = self.Name.location
except:
location = ['time','row','col']
try:
limits = self.name.qlocation
except:
try:
limits = self.Name.qlocation
except:
limits = set_default_limits(location)
# refl_check=False,names=None,\
# control=['mask','vza','vaa','sza','saa'],\
# location=['time','row','col'],limits=None
# location specifies the dimesions and names of the
# problem, e.g., & typically [time,row,col]
limits = np.array(check_limits_valid(limits))
try:
f = np.load(filename)
if not type(f).__name__ == 'NpzFile':
f.close()
self.error_msg="%s is not a NpzFile"%filename
self.error=True
if 'logger' in self or 'logger' in self.dict():
self.logger.info(self.error_msg)
return 0,(self.error,self.error_msg)
except:
self.error_msg="a problem opening %s as a NpzFile"%filename
self.error=True
if 'logger' in self or 'logger' in self.dict():
self.logger.info(self.error_msg)
return 0,(self.error,self.error_msg)
# ok so far then
# lets have a look inside
ncontents = np.array(f.files)
contents = np.array(f.files)
# translation table for default names
def_names = 'b1 b2 b3 b4 b5 b6 b7'.split()
if names == None:
# assume MODIS
names = def_names
def_alt_names = \
'645.5 856.5 465.6 553.6 1241.6 1629.1 2114.1'.split()
# look for any of names in contents
datasets = []
alt_datasets = []
alt_names = names
for i in xrange(len(np.atleast_1d(contents))):
if contents[i] in names:
datasets.append(i)
if not len(np.atleast_1d(datasets)):
if 'logger' in self or 'logger' in self.dict():
self.logger.error(\
"None of requested datasets %s found in %s ..." \
%(str(names),filename) + \
" trying default MODIS names: only %s"\
%(str(contents)))
names = def_names
alt_names = def_alt_names
for i in xrange(len(np.atleast_1d(contents))):
if contents[i] in names:
datasets.append(i)
if not len(np.atleast_1d(datasets)):
self.error_msg = "None of requested datasets %s found in %s"\
%(str(names),filename) + ' ' + \
"... trying default MODIS names: only %s"\
%(str(contents))
self.error = True
if 'logger' in self or 'logger' in self.dict():
self.logger.error(self.error_msg)
return 0,(self.error,self.error_msg)
trans_names = {}
for (i,j) in enumerate(alt_names):
trans_names[names[i]] = j
#trans_names = {names[i]:j for (i,j) in enumerate(alt_names)}
alt_name = []
this_name = []
for i in datasets:
this_name.append(contents[i])
alt_name.append(trans_names[contents[i]])
# Translate some old stylies...
trans = {'raa':'vaa','doys':'time'}
for i in trans:
if i in contents:
ncontents[np.where(contents==i)[0]]=trans[i]
# as a minimum, there needs to be some definition of one of
# the terms in location
# check how many dimensions this has
# now find a dataset
try:
# This could be more general, but this will do for now as its useful
# for spatial datasets
QA_OK = np.array(\
[8, 72, 136, 200, 1032, 1288, 2056,2120, 2184, 2248])
doy = f['doys'] - 2004000
qa = f['qa']
vza = f['vza']
sza = f['sza']
raa = f['raa']
y = []
for i in this_name:
y.append(f[i])
#mask = np.logical_or.reduce([qa==x for x in QA_OK ])
if 'logger' in self or 'logger' in self.dict():
self.logger.info(\
"sucessfully interpreted NpzFile dataset from %s"\
%filename)
self.logger.info("sub-setting ...")
controls = []
locations = []
grid = []
qlocations = []
thisshape = vza.shape
starter = {'time':np.min(doy),'row':0,'col':0}
delta = {'time':1,'row':1,'col':1}
if len(np.atleast_1d(limits)) <3:
from eoldas_Lib import set_default_limits
old_loc = location
location = np.array(['time','row','col'])
lim2 = set_default_limits(location)
for i in xrange(len(np.atleast_1d(limits))):
ww = np.where(old_loc[i] == location)[0]
lim2[ww] = list(limits[i])
limits = lim2
for i in xrange(len(np.atleast_1d(limits))):
if limits[i][0] == None:
limits[i][0] = starter[location[i]]
if limits[i][1] == None:
limits[i][1] = (thisshape[i]-1) + starter[location[i]]
if limits[i][2] == None:
limits[i][2]= delta[location[i]]
limits = np.array(limits)
start_doy = limits[0][0]
end_doy = limits[0][1]
step_doy = limits[0][2]
start_row = limits[1][0]
end_row = limits[1][1]
step_row = limits[1][2]
start_col = limits[2][0]
end_col = limits[2][1]
step_col = limits[2][2]
gooddays = np.logical_and.reduce(np.concatenate(\
([doy >= start_doy],[doy <=end_doy])))
qa = qa[gooddays,start_row:end_row+1,start_col:end_col+1]
vza = vza[gooddays,start_row:end_row+1,start_col:end_col+1]*0.01
sza = sza[gooddays,start_row:end_row+1,start_col:end_col+1]*0.01
raa = raa[gooddays,start_row:end_row+1,start_col:end_col+1]*0.01
yy = []
for i in xrange(len(np.atleast_1d(this_name))):
this = y[i]
yy.append(this[gooddays,start_row:end_row+1,\
start_col:end_col+1]*0.0001)
doy = doy[gooddays]
# now do QA
mask = np.zeros_like(qa).astype(bool)
# loop over qa
for j in xrange(len(np.atleast_1d(QA_OK))):
ww = np.where(qa==QA_OK[j])
mask[ww] = True
# better look over data to check valid
for j in xrange(len(np.atleast_1d(yy))):
ww = np.where(yy[j] < 0)
mask[ww] = False
ww = np.where(mask)
if 'logger' in self or 'logger' in self.dict():
self.logger.debug('parsing dataset: %d samples look ok'\
%np.array(ww).shape[1])
vza = vza[ww]
sza = sza[ww]
raa = raa[ww]
doy= doy[ww[0]]
row = ww[1]+start_row
col = ww[2]+start_col
locations = np.array([doy,row,col])
nnn = len(np.atleast_1d(locations[0]))
orig = np.repeat(np.array([start_doy,start_row,start_col]),locations.shape[1]).reshape(locations.shape).T
div = np.repeat(np.array([step_doy,step_row,step_col]),locations.shape[1]).reshape(locations.shape).T
qlocations = ((locations.T - orig)/div.astype(float)).astype(int).T
controls = np.array([np.ones_like(doy).astype(bool),\
vza,raa,sza,0*doy])
y = []
for i in xrange(len(np.atleast_1d(this_name))):
this = yy[i]
y.append(this[ww])
grid = np.array(y)
fmt = 'BRDF-UCL'
control = ['mask','vza','vaa','sza','saa']
bands = alt_name
if not np.array(grid).size:
if 'logger' in self or 'logger' in self.dict():
self.logger.error(\
"Warning: returning a zero-sized dataset ... "+\
" I wouldn;t try to do anything with it")
# in case we dont have data for all bands
mask = np.logical_or.reduce([[this_name[i]==x for x in names] \
for i in xrange(len(np.atleast_1d(this_name)))])
sd = np.array('0.004 0.015 0.003 0.004 0.013 0.01 0.006'\
.split())[mask]
sd = np.array([float(i) for i in sd.flatten()])\
.reshape(sd.shape)
nsamps = grid.shape[1]
sd = sd.repeat(nsamps).reshape(grid.shape).T
datasets = ParamStorage()
datasets.data = ParamStorage()
datasets.name = ParamStorage()
datasets.name.fmt = fmt
grid = grid.T
datasets.data[name] = np.zeros([grid.shape[0],len(np.atleast_1d(names))])\
.astype(object)
datasets.data[name][:,:] = None
for i in xrange(len(np.atleast_1d(this_name))):
ww = np.where(names == this_name[i])[0][0]
datasets.data[name][:,ww] = grid[:,i]
datasets.data.location = np.array(locations).T
datasets.data.control = np.array(controls).T
datasets.data.qlocation = np.array(qlocations).T
datasets.name[name] = np.array(names)
datasets.name.location = np.array(['time','row','col'])
datasets.name.control = np.array(control)
datasets.name.qlocation = limits
datasets.name.bands = np.array(bands)
datasets.data.sd = np.zeros([grid.shape[0],len(np.atleast_1d(names))])\
.astype(object)
# for i in xrange(grid.shape[0]):
# datasets.data.sd[i,:] = self.options.sd
datasets.data.sd[:,:] = None
for i in xrange(len(np.atleast_1d(this_name))):
ww = np.where(names == this_name[i])[0][0]
datasets.data.sd[:,ww] = sd[:,i]
datasets.name.sd = np.array(names)
if 'logger' in self or 'logger' in self.dict():
self.logger.debug('finished parsing dataset')
except:
self.error_msg=\
"a problem processing information from %s as a NpzFile"\
%filename
self.error=True
if 'logger' in self or 'logger' in self.dict():
self.logger.info(self.error_msg)
return 0,(self.error,self.error_msg)
f.close()
if 'logger' in self or 'logger' in self.dict():
self.logger.info('... done')
self.error=False
self.error_msg=""
return datasets,(self.error,self.error_msg)
def demonstration():
from eoldas_State import State
from eoldas_ParamStorage import ParamStorage
import numpy as np
# a basic set up for State, setting names & bounds etc
options = ParamStorage()
options.logfile = 'test/data_type/logs/log.dat'
options.names = \
'gamma xlai xhc rpl xkab scen xkw xkm xleafn xs1 xs2 xs3 xs4 lad'.split()
options.bounds = [[0.01,None],\
[0.01,0.99],\
[0.01,10.0],\
[0.001,0.10],\
[0.1,0.99],\
[0.0,1.0],\
[0.01,0.99],\
[0.3,0.9],\
[0.9,2.5],\
[0.0, 4.],\
[0.0, 5.],\
[None, None],\
[None, None],\
[None, None]]
options.default = -1.0 * np.ones(len(options.names))
options.location = 'time'.split()
options.control = 'mask vza vaa sza saa'.split()
options.datadir = ['.', 'test/data_type']
name = "eoldas_data_type test 0"
options.limits = [[170, 365, 1]]
self = State(options,datatype='y',name=name,datadir=\
options.datadir,env=None,logfile=options.logfile)
self.tester()
# Now we set some state data
this = ParamStorage()
# how many state vector elements should there be?
n_states = len(self.Name.state)
self.state = np.ones([100, n_states])
self.Data.sd = np.ones([1, n_states])
self.Name.sd = self.Name.state
print '******************'
print(self.Data.sd, self.Name.sd)
this.data = ParamStorage()
this.name = ParamStorage()
this.data.state = self.state * 2
controls = self.Name.control
n_controls = len(controls)
this.data.control = np.ones([100, n_controls])
this.data.location = np.ones([100, n_controls])
# we can load x_state from a ParamStorage
self.state = this
# now we should see the control data etc.
self.tester()
# change a dataset name to see if that works:
# should load everything as numpy arrays
self.Name.control = np.array(['vza', 'sza'])
# change a dataset to see if that works .. deliberately load a bad one
self.Data.control = 0
# now try a direct state data load
self.state = np.zeros([100, 100]) + 5.
# which will load into self.Data.state
self['state'] = np.zeros([100, 100]) + 6.
#now try accessing it:
print self.state
# change the control info
self.Name.control = np.array(['vza'])
print self.Name.control
# reset it
self.state = self.state * 2
print self.state
# now try reading a file into state
self.state = 'test/data_type/input/test.brf'
print '========'
print 'data from a BRDF file'
self.tester()
print '========'
# written as a pickle
self.write('test/data_type/output/test.pickle', None, fmt='pickle')
self.logger.info("...DONE...")
name = "eoldas_data_type test 1"
del self
self1 = State(options,datatype='y',name=name,datadir=\
options.datadir,env=None,logfile=options.logfile,grid=True)
# read from pickle
self1.state = 'test/data_type/output/test.pickle'
print '========'
print 'data from a pickle file'
self1.tester()
print '========'
# try to load an npx file
del self1
options.location = 'time row col'.split()
options.limits = [[170, 365, 1], [0, 500, 1], [200, 200, 1]]
self2 = State(options,datatype='y',name=name,datadir=\
options.datadir,env=None,logfile=options.logfile)
self2.state = 'test/data_type/input/interpolated_data.npz'
print '========'
print 'data from a npz file'
self2.tester()
print '========'
# write as BRDF-UCL
self2.write('test/data_type/output/test.brf', None, fmt='BRDF-UCL')
del self2
self3 = State(options,datatype='y',name=name,datadir=\
options.datadir,env=None,logfile=options.logfile)
# then test the reader
self3.state = 0.
self3.state = 'test/data_type/output/test.brf'
print '========'
print 'data from a BRDF-UCL file'
print '========'
self3.tester()
print '========'
# then write as a PARAMETERS file
self3.write('test/data_type/output/test.param', None, fmt='PARAMETERS')
del self3
options.location = 'time row col'.split()
options.limits = [[170, 365, 1], [0, 500, 1], [200, 200, 1]]
options.control = np.array(['mask', 'vza', 'vaa', 'sza', 'saa'])
self4 = State(options,datatype='y',name=name,datadir=\
options.datadir,env=None,logfile=options.logfile)
# then test the reader
self4.state = 0.
self4.state = 'test/data_type/output/test.param'
print '========'
print 'data from a PARAMETERS file'
print '========'
self4.tester()
print '========'
