def process_data_string(self, name, info=[], fmt=None):
'''
Attempt to load data from a string, assuming the string is a filename.
The array self.datadir is searched for readable files with the
string 'name' (also self.env), and a list of potential files
considered for reading.
Each readable file is passed to self.read, and if it is interpretable,
it is loaded according to the read method.
Note tha the format can be specified. If not, then all formats
are attempted until a sucessful read is made.
'''
from eoldas_Lib import get_filename
orig = self.data[name]
if self.logger:
self.logger.debug('%s is a string ... see if its a readable file ...' \
% name)
# find a list of potential files
goodfiles, is_error = get_filename(orig,datadir=self.datadir,\
env=self.env,multiple=True)
if is_error[0] and self.logger:
self.logger.debug(str(is_error[1]))
return
if self.logger:
self.logger.debug("*** looking at potential files %s" %
str(goodfiles))
# loop over all files that it might be
for goodfile in goodfiles:
stuff, is_error = reader(self, goodfile, name, fmt=fmt, info=info)
if not is_error[0] and self.logger:
self.logger.info("Read file %s " % goodfile)
return
if self.logger:
self.logger.debug(self.error_msg)
return
def process_data_string(self,name,info=[],fmt=None):
'''
Attempt to load data from a string, assuming the string is a filename.
The array self.datadir is searched for readable files with the
string 'name' (also self.env), and a list of potential files
considered for reading.
Each readable file is passed to self.read, and if it is interpretable,
it is loaded according to the read method.
Note tha the format can be specified. If not, then all formats
are attempted until a sucessful read is made.
'''
from eoldas_Lib import get_filename
orig = self.data[name]
if self.logger:
self.logger.debug('%s is a string ... see if its a readable file ...' \
% name)
# find a list of potential files
goodfiles, is_error = get_filename(orig,datadir=self.datadir,\
env=self.env,multiple=True)
if is_error[0] and self.logger:
self.logger.debug(str(is_error[1]))
return
if self.logger:
self.logger.debug("*** looking at potential files %s"%str(goodfiles))
# loop over all files that it might be
for goodfile in goodfiles:
stuff,is_error = reader(self,goodfile,name,fmt=fmt,info=info)
if not is_error[0] and self.logger:
self.logger.info("Read file %s "%goodfile)
return
if self.logger:
self.logger.debug(self.error_msg)
return
def read_single_conf_file (self, conf_files,options=None):
"""
Purpose:
parse the information from conf_files into a
ConfigParser class instance and return this.
Parameters:
conf_files : list of one or more config files
Options:
options=None : pass an options structure through
Uses:
self.datadir=['.',,'~/.eoldas']
: list of directories to look
for config files
self.env=None : name of an environment variable
where config files
can be searched for if not found
in datadir (or absolute
path name not given)
self.fatal=False : flag to state whether the
call should fail if
a requested config file is not found.
Returns:
tuple : (config, config_error)
where:
config : ConfigParser class instance
or False if an error occurs
config_error : string giving information on error
"""
import ConfigParser
from eoldas_Lib import get_filename
# Instantiate a parser
config = ConfigParser.ConfigParser()
# Read the config files. If it doesn't exist, raise exception.
#
if type(conf_files) == str:
conf_files = [conf_files]
all_conf_files = []
for fname in conf_files:
fname,fname_err = get_filename(fname,datadir=self.datadir,\
env=self.env)
if fname_err[0] != 0:
if self.fatal:
return False,False,\
"Cannot find configuration file %s\n%s" \
% (fname,fname_err[1])
else:
all_conf_files.append(fname)
thisdir = os.path.dirname(fname)
if not thisdir in self.datadir:
self.datadir.append(thisdir)
if len(all_conf_files) == 0:
return False,False,\
"%s: No valid conf files found in list %s in dirs %s" \
% (os.getcwd(),conf_files,self.datadir)
config.config_files = config.read(all_conf_files)
if len(config.config_files) == 0:
return False,False,\
"%s: No valid conf files found in list %s in dirs %s" \
% (os.getcwd(),conf_files,self.datadir)
# from here on, we attempt to pull specific information from
# the conf files
info = ParamStorage(name='info',doc=\
'Configuration information for %s' % \
str(config.config_files))
# scan everything into config.info
# but it helps to sort it to get the info in the right order
sections = config.sections()
#sections.sort()
firstsections = []
secondsections = []
for this in sections:
if this[:7] == 'general' or this[:9] == 'parameter':
firstsections.append(this)
else:
secondsections.append(this)
firstsections.sort()
sections = firstsections
[sections.append(i) for i in secondsections]
for this in sections:
self.logger.debug('...Section %s'%this)
self.scan_info(config,this,info,this,info)
self.rescan_info(config,this,info,this,info,0)
self.config = config
self.info = info
if options != None and type(options) == ParamStorage:
self.info.update(options,combine=True)
# sort any helper text looping over self.info
# into self.loaders
self.__sort_help(self.info,"")
try:
self.logger.info("Config: %s read correctly" \
% str(all_conf_files))
except:
pass
return self.config,self.info,"Config: %s read correctly" \
% str(all_conf_files)
def read_single_conf_file(self, conf_files, options=None):
"""
Purpose:
parse the information from conf_files into a
ConfigParser class instance and return this.
Parameters:
conf_files : list of one or more config files
Options:
options=None : pass an options structure through
Uses:
self.datadir=['.',,'~/.eoldas']
: list of directories to look
for config files
self.env=None : name of an environment variable
where config files
can be searched for if not found
in datadir (or absolute
path name not given)
self.fatal=False : flag to state whether the
call should fail if
a requested config file is not found.
Returns:
tuple : (config, config_error)
where:
config : ConfigParser class instance
or False if an error occurs
config_error : string giving information on error
"""
import ConfigParser
from eoldas_Lib import get_filename
# Instantiate a parser
config = ConfigParser.ConfigParser()
# Read the config files. If it doesn't exist, raise exception.
#
if type(conf_files) == str:
conf_files = [conf_files]
all_conf_files = []
for fname in conf_files:
fname,fname_err = get_filename(fname,datadir=self.datadir,\
env=self.env)
if fname_err[0] != 0:
if self.fatal:
return False,False,\
"Cannot find configuration file %s\n%s" \
% (fname,fname_err[1])
else:
all_conf_files.append(fname)
thisdir = os.path.dirname(fname)
if not thisdir in self.datadir:
self.datadir.append(thisdir)
if len(all_conf_files) == 0:
return False,False,\
"%s: No valid conf files found in list %s in dirs %s" \
% (os.getcwd(),conf_files,self.datadir)
config.config_files = config.read(all_conf_files)
if len(config.config_files) == 0:
return False,False,\
"%s: No valid conf files found in list %s in dirs %s" \
% (os.getcwd(),conf_files,self.datadir)
# from here on, we attempt to pull specific information from
# the conf files
info = ParamStorage(name='info',doc=\
'Configuration information for %s' % \
str(config.config_files))
# scan everything into config.info
# but it helps to sort it to get the info in the right order
sections = config.sections()
#sections.sort()
firstsections = []
secondsections = []
for this in sections:
if this[:7] == 'general' or this[:9] == 'parameter':
firstsections.append(this)
else:
secondsections.append(this)
firstsections.sort()
sections = firstsections
[sections.append(i) for i in secondsections]
for this in sections:
self.logger.debug('...Section %s' % this)
self.scan_info(config, this, info, this, info)
self.rescan_info(config, this, info, this, info, 0)
self.config = config
self.info = info
if options != None and type(options) == ParamStorage:
self.info.update(options, combine=True)
# sort any helper text looping over self.info
# into self.loaders
self.__sort_help(self.info, "")
try:
self.logger.info("Config: %s read correctly" \
% str(all_conf_files))
except:
pass
return self.config,self.info,"Config: %s read correctly" \
% str(all_conf_files)
def load_bandpass_library ( bandpass_filename, bandpass_library={}, \
pad_edges=True ):
"""Loads bandpass files.
Loads bandpass files in ASCII format. The data are read from
`bandpass_filename`, the spectral range of interest is `nlw`
(usually in nm) and if you have already a dictionary with named
bandpass functions, you can give it as `bandpass_library`.
Note that bands names that are already present
in the library will be ignored. If the bandpass functions aren't
padded to 0 at the edges of the band, the `pad_edges` option will
set them to 0 at the edges. Otherwise, the interpolation goes a
bit crazy.
A bandpass file contains something like:
[begin MODIS-b2]
450 0.0
550 1.0
650 2.0
750 1.0
850 0.0
[end MODIS-b2]
Npte that it doesn't need to be normalised when defined.
"""
from eoldas_Lib import get_filename
fname,fname_err = get_filename(bandpass_filename,datadir=self.datadir)
if fname_err[0] == 0:
bp_fp = open ( fname, 'r' )
else:
return bandpass_library,fname_err
self.logger.info('file %s loaded into bandpass library'%fname)
nlw = self.nlw
data = bp_fp.readlines()
bp_fp.close()
bands = {}
for ( i, dataline ) in enumerate ( data ):
if dataline.find ( "[begin" ) >= 0:
name_start = dataline.replace("[begin", "").strip().\
replace("]", "")
bands[ name_start ] = []
elif dataline.find ( "[end" ) >= 0:
name_end = dataline.replace("[end", "").strip().replace("]", "")
# Make sure the file is consistent
if name_end != name_start:
fname_err[0] = True
fname_err[1] = \
"Inconsistent '[begin ]' and " + \
"'[end]' names (%s != %s) at line %d" \
% (name_start, name_end, i+1 )
return bandpass_library,fname_err
name_start = None
else:
( x, y ) = dataline.strip().split()
# Double check ehtat we have a good band name
if name_start is not None:
fname_err[0] = True
fname_err[1] = \
"Bandpass file appears corrupt " + \
"at line " % (i+1)
return bandpass_library,fname_err
bands[name_start].append ( [float(x), float(y)] )
for ( band_name, filter_fncn ) in bands.iteritems() :
if not bandpass_library.has_key( band_name ):
filter_fncn = np.array ( filter_fncn )
if pad_edges:
bandpass_library [ band_name ] = np.interp ( nlw, \
filter_fncn[:,0], filter_fncn[:, 1], left=0.0, right=0.0)
else:
bandpass_library [ band_name ] = np.interp ( nlw, \
filter_fncn[:,0], filter_fncn[:, 1] )
return bandpass_library,True
