class SpecialVariable(ParamStorage):
'''
A class that can deal with the datatypes needed for eoldas
It allows a variable to be set to various data types
and interprets these into the data structure.
The data structure imposed here is:
self.data.this
self.name.this
to store some item called 'this'. Other information can be stored as well
but part of the idea here is to have some imposed constraints on the
data structure so that we can sensibly load up odfferent datasets.
The idea is that data will be stored in self.data and associated
metadata in self.name. If items are given the same name in both
sub-structures, we can easily keep track of them. There is no actual
requirement that this is adhered to, but it is certainy permitted and
encouraged for the indended use of this class.
Probably the most important thing then about this class is that is
a SpecialVariable is assigned different data types, then it can do sensible
things with them in the context of the EOLDAS (and wider applications).
When an assignment takes place (either of ther form
self.state = foo
or
self['state'] = foo
then what is actually stored depends on the type and nature of foo. The
main features as follows:
If foo is a string:
A guess is made that it is a filename, and an attempt is made to
read the file. All directories in the list self.dirnames are
searched for the filename, and any readable files found are
considered candidates, Each of these is read in turn. A set of
potential data formats, specified by the readers in readers
(self.reader_functions) is considered, as if a sucessful interpretation
takes place the data is returned and stiored in the derired variable.
So, for example, if we have self.state = foo as above and foo is a valid,
readable file in the list of directories specified, and it is
interprable with one of the formats defined, then the main dataset
is loaded into:
self.data.state
(alternatively known as self.data['state']).
If foo is a ParamStorage, it should have the same structure as that here
(i.e. self.data and self.name) and these structures are then loaded.
If foo is a dictionary (type dict)
It is first converted to a ParamStorage and then loaded as above.
If foo is any other datatype, it is left pretty much as it, except that
an attempt to convert to a np.array is made.
Depending on the format, there might be data other than the main
dataset (e.g. locational information) and these are loaded by the
loaders into relevant oarts of self.data and self.name.
For classes that use this class for EOLDAS, we will typically use:
self.data.state : state variable data
self.data.sd : uncertainty information as sd
(or a similar fuller representation)
self.data.control : control information (e.g. view angles)
self.data.location : location information
with associated descriptor data in the relevant parts of self.name.
The idea for simple use of the data structure then is for all of these
datasets represented as 2D datasets, where the number of rows in
each of the self.data.state etc field will be the same, but
the number of columns will tend to vary (e.g different numbers of state
variables).
The reason for considering such a 2D 'flat'(ish) representation is
that it is easy to tabulate and understand. In fact the data will be of
quite high dimension. E.g. is the data vary by time, x and y, then
we would have 3 columns for self.data.location, with descriptors for
the columns in self.name.location, and corresponding state data in
self.data.state (with the number of state variables determining the
number of columns in that table).
As mentioned, at the pooint of this class, there is no strict
requirement for any such structure ti the data loaded or used, but
that is the plan for EOLDAS use, so worth documenting at this point.
'''
def __init__(self,info=[],name=None,thisname=None,readers=[],log_terms={},\
datadir=["."],env=None,\
header=None,writers={},
simple=False,logger=None):
'''
Class SpecialVariable initialisation.
Sets up the class as a ParamStorage and calls self.init()
See init() fort a fuller descripotion of the options.
'''
ParamStorage.__init__(self,logger=logger)
name = name or thisname
if name == None:
import time
thistime = str(time.time())
name = type(self).__name__
name = "%s.%s" % (name,thistime)
self.thisname = name
self.init(info=[],name=self.thisname,readers=readers,log_terms={},\
datadir=datadir,env=env,\
header=header,writers=writers,\
simple=False,logger=logger)
def init(self,info=[],name=None,readers=[],log_terms={},\
datadir=None,env=None,\
header=None,writers={},\
simple=False,logger=None):
'''
Initialise information in a SpecialVariable instance.
Options:
info : Information tthat can be passed through to reader
methods (a list).
thisname : a name to use to identify this instance in any
logging. By default this is None. If thisname is set
to True, then logging is to stdout.
readers : A list of reader methods that are pre-pended to
those already contained in the class.
log_terms : A dictionary of log options. By default
{'logfile':None,'logdir':'log','debug':True}
If thisname is set, and logfile specified, then
logs are logged to that file. If thisname is set
to True, then logging is to stdout.
datadir : A list of directories to search for data files
to interpret if the SpecialVariable is set to a
string.
env : An environment variable that can be used to extend
the datadir variable.
header : A header string to use to identify pickle files.
By default, this is set to
"EOLDAS -- plewis -- UCL -- V0.1"
simple : A flag to swicth off the 'complicated'
interpretation methods, i.e. just set and return
variables literally, do not try to interpret them.
'''
self.set('simple',True)
if name == None:
import time
thistime = str(time.time())
name = type(self).__name__
name = "%s.%s" % (name,thistime)
self.thisname = name
# this is where we will put any data
self.data = ParamStorage()
self.name = ParamStorage()
self.info = info
self.datadir = datadir or ['.']
self.env = env
init_read_write(self,header,readers,writers)
# sort logging and log if thisname != None
self.log_terms = {'logfile':None,'logdir':'log','debug':True}
# override logging info
for (key,value) in log_terms.iteritems():
self.log_terms[key] = value
self.logger= sortlog(self,self.log_terms['logfile'],logger,name=self.thisname,\
logdir=self.log_terms['logdir'],\
debug=self.log_terms['debug'])
self.simple = simple
set = lambda self,this,value :ParamStorage.__setattr__(self,this,value)
set.__name__ = 'set'
set.__doc__ = """
A method to set the literal value of this, rather than attempt
an interpretation (e.g. used when self.simple is True)
"""
get = lambda self,this :ParamStorage.__getattr__(self,this)
get.__name__ = 'get'
get.__doc__ = """
A method to get the literal value of this, rather than attempt
an interpretation (e.g. used when self.simple is True)
"""
def __setitem__(self,this,value):
'''
Variable setting method for style self['this'].
Interpreted the same as via __setattr__.
'''
# always set the item
self.__setattr__(this,value)
def __setattr__(self,this,value):
'''
Variable setting method for style self.this
Varies what it does depending on the type of value.
The method interprets and sets the SpecialVariable value:
1. ParamStorage or SpecialVariable. The data are directly loaded.
This is one of the most flexible formats for input. It expects
fields 'data' and/or 'name', which are loaded into self.
There will normally be a field data.this, where this is
the variable name passed here.
2. A dictionary, same format as the ParamStorage.
3. A tuple, interpreted as (data,name) and loaded accordingly.
4. *string* as filename (various formats). An attempt to read the
string as a file (of a set of formats) is made. If none pass
then it it maintained as a string.
5. A numpy array (np.array) that is loaded into self.data.this.
6. Anything else. Loaded into self.data.this as a numpy array.
'''
if self.simple:
self.set(this,value)
return
t = type(value)
try:
if t == ParamStorage or t == SpecialVariable:
# update the whole structure
#self.__set_if_unset('data',ParamStorage())
#self.__set_if_unset('name',ParamStorage())
self.data.update(value.data,combine=True)
self.name.update(value.name,combine=True)
elif t == dict:
n_value = ParamStorage().from_dict(value)
self.__setattr__(this,n_value)
elif t == tuple or t == list:
# assumed to be (data,name) or [data,name]
#self.__set_if_unset('data',ParamStorage())
#self.__set_if_unset('name',ParamStorage())
#ParamStorage.__setattr__(self['data'],this,value[0])
#ParamStorage.__setattr__(self['name'],this,value[1])
ParamStorage.__setattr__(self['data'],this,np.array(value))
elif t == str:
# set the term
#self.__set_if_unset('data',ParamStorage())
#self.__set_if_unset('name',ParamStorage())
ParamStorage.__setattr__(self['data'],this,value)
# interpret as a file read if possible
self.process_data_string(this,info=self.info)
elif t == np.ndarray:
#self.__set_if_unset('data',ParamStorage())
#self.__set_if_unset('name',ParamStorage())
ParamStorage.__setattr__(self['data'],this,value)
else:
ParamStorage.__setattr__(self['data'],this,\
np.array(value))
except:
if self.logger:
self.logger.info("Failed to set SpecialVariable %s from %s %s"\
%(this,t.__name__,value))
return
if self.logger:
self.logger.info("Set variable %s from type %s"%(this,t.__name__))
def __getattr__(self,name):
'''
Variable getting method for style self.this
If the field 'data' exists in self.__dict__ and 'name'
is in the dictionary, then the field self.data.this is returned.
Otherwise, if the field 'name' is in self.__dict__, self.name
is returned.
Otherwise return None.
'''
if 'data' in self.__dict__ and name in self.data.__dict__:
return self.data.__dict__.__getitem__ ( name )
elif name in self.__dict__:
return self.__dict__.__getitem__ ( name )
else:
return None
def __getitem__(self,name):
'''
Variable getting method for style self['this'].
Interpreted the same as via __getattr__.
'''
# first look in data
if 'data' in self.__dict__ and name in self.data.__dict__:
return self.data.__dict__.__getitem__ ( name )
elif name in self.__dict__:
return self.__dict__.__getitem__ ( name )
else:
return None
def process_data_string(self,name,info=[],fmt=None):
'''
Attempt to load data from a string, assuming the string is a filename.
The array self.datadir is searched for readable files with the
string 'name' (also self.env), and a list of potential files
considered for reading.
Each readable file is passed to self.read, and if it is interpretable,
it is loaded according to the read method.
Note tha the format can be specified. If not, then all formats
are attempted until a sucessful read is made.
'''
from eoldas_Lib import get_filename
orig = self.data[name]
if self.logger:
self.logger.debug('%s is a string ... see if its a readable file ...' \
% name)
# find a list of potential files
goodfiles, is_error = get_filename(orig,datadir=self.datadir,\
env=self.env,multiple=True)
if is_error[0] and self.logger:
self.logger.debug(str(is_error[1]))
return
if self.logger:
self.logger.debug("*** looking at potential files %s"%str(goodfiles))
# loop over all files that it might be
for goodfile in goodfiles:
stuff,is_error = reader(self,goodfile,name,fmt=fmt,info=info)
if not is_error[0] and self.logger:
self.logger.info("Read file %s "%goodfile)
return
if self.logger:
self.logger.debug(self.error_msg)
return
write = lambda self,filename,fmt : writer(self,filename,None,fmt=fmt)
read = lambda self,filename,fmt : reader(self,filename,None,fmt=fmt,info=[])
class Parser():
def __init__(self,args,name=None,general=None,log=False,logger=None,outdir=".",getopdir=False,parse=True):
"""
Initialise parser class.
This sets up the class defaults.
Options:
general=general: this over-rides and defaults with values
set in parser general can be of the form:
1. class ParamStorage (i.e. the same form
as self.general)
2. a command line list (where the first
item in the list is ignored
3. a string containing a set of command line general
See self.parse() for more details on general
2 and 3 as these simply make a call to tha
method.
log=True If log is set to True, then logging starts
when this class is instanced.
Note that the logfile and logdir might
change if subsequent calls to Parser.parse()
are made
"""
if type(args) == str:
args = args.split()
self.dolog = log
self.log = log
self.name = args[0]
self.args = args[1:]
self.fullargs = args
self.store_fullargs = args
if name == None:
import time
thistime = str(time.time())
name = type(self).__name__
name = "%s.%s" % (name,thistime)
self.thisname = name
# find the following flags:
# --conf | -c : conf
# --datadir : datadir
datadir = [".","~/.eoldas",sys.path[0]+'/../bin',sys.path[0]+'/../confs',\
sys.path[0]+'/../system_confs',sys.path[0]+'/../eoldaslib']
conf = "default.conf"
logfile = None
logdir = "."
self.top = ParamStorage ()
self.top.general = ParamStorage ()
self.top.general.__helper__ = ParamStorage ()
self.top.general.__default__ = ParamStorage ()
self.top.general.__extras__ = ParamStorage ()
self.top.general.conf = []
for i in xrange(len(self.args)):
theseargs = self.args[i].split('=')
if theseargs[0] == "--conf":
conf = theseargs[1]
self.top.general.conf.append(conf)
elif theseargs[0][0:2] == "-c":
if len(theseargs) > 2:
conf = theseargs[0][2:]
else:
conf = self.args[i+1]
self.top.general.conf.append(conf)
elif theseargs[0] == "--datadir":
datadir1 = theseargs[1].replace('[','').\
replace(']','').split()
[datadir1.append(datadir[i]) for i in \
xrange(len(datadir))]
datadir = datadir1
elif theseargs[0] == "--logfile":
logfile= theseargs[1]
elif theseargs[0] == "--logdir":
logdir = theseargs[1]
elif theseargs[0] == "--outdir":
outdir = theseargs[1]
if self.top.general.conf == []:
self.top.general.conf = conf
if logfile == None:
logfile = conf.replace('conf','log')
self.top.general.here = os.getcwd()
self.top.general.datadir = datadir
self.top.general.logfile = logfile
self.top.general.logdir = logdir
self.top.general.outdir = outdir
# add here to datadir
# in addition to '.' to take account of the change of directory
self.top.general.datadir = self.__add_here_to_datadir(\
self.top.general.here,self.top.general.datadir)
self.top.general.datadir = self.__add_here_to_datadir(\
self.top.general.outdir,self.top.general.datadir)
# cd to where the output is to be
self.__cd(self.top.general.outdir)
# set up the default command line options
self.default_loader()
# update with anything passed here
if general and type(general) == ParamStorage: self.top.update(\
self.__unload(general),combine=True)
# read the conf files to get any cmd line options
self.logger = sortlog(self,self.top.general.logfile,logger,name=self.thisname,\
logdir=self.top.general.logdir)
self.config = ConfFile(self.top.general.conf,name=self.thisname+'.config',\
loaders=self.loaders,datadir=self.top.\
general.datadir,logger=self.logger,logdir=self.top.general.logdir,\
logfile=self.top.general.logfile)
if len(self.config.configs) == 0:
this = "Warning: Nothing doing ... you haven't set any configuration",\
self.config.storelog
try:
self.logger(this)
except:
print "Called with args:"
print "eoldas",self.args
pass
raise Exception(this)
# now loaders contains all of the defaults set here
# plus those from the config (config opver-rides defaults here)
self.loaders = self.config.loaders
# now convert loaders into parser information
self.parseLoader(self.loaders)
self.parse(self.fullargs)
if general and type(general) == ParamStorage:
self.top.update(self.__unload(general),combine=True)
if general and type(general) == str:
self.parse(general.split())
if general and type(general) == list:
self.parse(general)
# now update the info in self.config
for i in self.config.infos:
i.update(self.top,combine=True)
# so now all terms in self.config.infos
# contain information from the config file, updated by
# the cmd line
i.logger = self.logger
i.log()
# move the information up a level
self.infos = self.config.infos
self.configs = self.config.configs
self.config_log = self.config.storelog
#if getopdir:
# self.sortnames()
self.config.loglist(self.top)
#del(self.config.infos)
#del(self.config.configs)
#del(self.config)
self.__cd(self.top.general.here)
def __add_here_to_datadir(self,here,datadir):
from os import sep,curdir,pardir
if type(datadir) == str:
datadir = [datadir]
iadd = 0
for i in xrange(len(datadir)):
j = i + iadd
if datadir[j] == curdir or datadir[j] == pardir:
tmp = datadir[:j]
rest = datadir[j+1:]
tmp.append("%s%s%s" % (here,sep,datadir[j]))
tmp.append(datadir[j])
iadd += 1
for k in xrange(len(rest)):
tmp.append(rest[k])
datadir = tmp
return datadir
def default_loader(self):
"""
Load up parser information for first pass
"""
self.loaders = []
self.top.general.here = os.getcwd()
self.loaders.append(["datadir",['.',self.top.general.here],\
"Specify where the data and or conf files are"])
self.loaders.append(["passer",False,\
"Pass over optimisation (i.e. report and plot the initial values)"])
self.loaders.append(["outdir",None,\
"Explicitly mspecify the results and processing output directory"])
self.loaders.append(["verbose",False,"Switch ON verbose mode","v"])
self.loaders.append(["debug",False,optparse.SUPPRESS_HELP,"d"])
self.loaders.append(["conf","default.conf",\
"Specify configuration file. Set multiple files by using the flag multiple times.","c"])
self.loaders.append(["logdir","logs",\
"Subdirectory to put log file in"])
self.loaders.append(["logfile","logfile.logs","Log file name"])
def parseLoader(self,loaders):
"""
Utility to load a set of terms from the list loaders
into the ParamStorage general
If there are 3 terms in each loaders element, they refer to:
1. name
2. default value
3. helper text
If there is a fourth, it is associated with extras
(short parser option)
"""
general = ParamStorage ()
general.__default__ = ParamStorage ()
general.__extras__ = ParamStorage ()
general.__helper__ = ParamStorage ()
for this in loaders:
if len(this) > 1:
general[this[0]] = this[1]
general.__default__[this[0]] = this[1]
else:
general[this[0]] = None
general.__default__[this[0]] = None
if len(this) > 2:
general.__helper__[this[0]] = this[2]
else:
general.__helper__[this[0]] = optparse.SUPPRESS_HELP
if len(this) > 3:
general.__extras__[this[0]] = "%s" % this[3]
else:
general.__extras__[this[0]] = None
# make sure arrays arent numpy.ndarray
if type(general.__default__[this[0]]) == np.ndarray:
general.__default__[this[0]] = \
list(general.__default__[this[0]])
self.top.update(self.__unload(general),combine=True)
def __list_to_string__(self,thisstr):
"""
Utility to convert a list to some useable string
"""
return(str(thisstr).replace('[','_').strip("']").\
replace('.dat','').replace("_'","_").replace(",","").\
replace(" ","").replace("''","_").replace("___","_").\
replace("__","_"));
def __cd(self,outdir):
if not os.path.exists(outdir):
try:
os.makedirs(outdir)
except OSerror:
print "Fatal: Prevented from creating",outdir
sys.exit(-1)
try:
os.chdir(outdir)
except:
print "Fatal: unable to cd to",outdir
raise Exception("Fatal: unable to cd to %s"%outdir)
def sortnames(self):
"""
Utility code to sort out some useful filenames & directories
"""
if self.top.general.outdir == None:
basename = self.top.general.basename
confnames = self.__list_to_string__(self.top.general.conf)
self.top.general.outdir = basename + "_conf_" + confnames
self.__cd(self.top.general.outdir)
def parse(self,args,log=False):
'''
Given a list such as sys.argv (of that form)
or an equivalent string parse the general and store
in self.parser
'''
self.dolog = log or self.dolog
if type(args) == type(""):
args = args.split()
args = args[1:]
self.top.general.cmdline = str(args)
usage = "usage: %prog [general] arg1 arg2"
parser = OptionParser(usage,version="%prog 0.1")
# we go through items in self.top.general and set up
# parser general for each
for this in sorted(self.top.general.__helper__.__dict__.keys()):
# sort out the action
# based on type of the default
default=self.top.general.__default__[this]
action="store"
thistype=type(default)
if thistype == type(None):
thistype = type("")
argss = '--%s'%this
dest = "%s"%this
helper = self.top.general.__helper__[this]
if type(default) == type([]):
# list, so append
action="store"
elif type(default) == type(True):
action="store_true"
typer = "string"
if thistype != type([]) and thistype != type(True):
typer='%s' % str(thistype).split("'")[1]
# has it got extras?
if self.top.general.__extras__[this] != None:
parser.add_option('-%s'%self.top.general.__extras__[\
this].lower(),argss,type="string",action=action,\
help=helper,default=str(default))
else:
parser.add_option(argss,action=action,help=helper,\
type="string",default=str(default))
elif ( thistype != type(True)):
if self.top.general.__extras__[this] != None:
parser.add_option('-%s'%self.top.general.__extras__[\
this].lower(),argss,type="string",action=action,\
help=helper,default=str(default))
else:
parser.add_option(argss,action=action,help=helper,\
default=str(default))
if thistype == type(True):
if self.top.general.__extras__[this] != None:
parser.add_option('-%s'%self.top.general.__extras__[\
this].lower(),argss,dest=dest,action=action,\
help=helper,default=default)
else:
parser.add_option(argss,action=action,help=helper,\
dest=dest,default=default)
that = this.split('.')
argss = '--'
for i in xrange(len(that)-1):
argss = argss + "%s." % that[i]
argss = argss + 'no_%s' % that[-1]
helper='The opposite of --%s'%this
action='store_false'
typer='%s' % str(thistype).split("'")[1]
# has it got extras?
if self.top.general.__extras__[this] != None:
parser.add_option('-%s'%self.top.general.__extras__[\
this].capitalize(),argss,dest=dest,action=action,\
help=helper)
else:
parser.add_option(argss,action=action,dest=dest,\
help=helper)
# we have set all option types as str, so we need to interpret
# them in__unload
(general, args) = parser.parse_args(args)
#for data_file in args:
# general.data_file.append(data_file)
#general.data_file = list(np.array(general.data_file).flatten())
#general.brf= list(np.array(general.brf).flatten())
# load these into self.general
self.top.update(self.__unload(general.__dict__),combine=True)
#self.sortnames()
if self.dolog:
self.log = set_up_logfile(self.top.general.logfile,\
logdir=self.top.general.logdir)
self.log_report()
def __unload(self,options):
from eoldas_ConfFile import array_type_convert
this = ParamStorage()
this.general = ParamStorage()
for (k,v) in options.iteritems():
ps = this
that = k.split('.')
if len(that) == 1:
ps = this.general
else:
for i in xrange(len(that)-1):
if not hasattr(ps,that[i]):
ps[that[i]] = ParamStorage()
ps = ps[that[i]]
# set the value v which needs to
# to be interpreted
ps[that[-1]] = array_type_convert(self.top,v)
return this
class Parser():
def __init__(self,
args,
name=None,
general=None,
log=False,
logger=None,
outdir=".",
getopdir=False,
parse=True):
"""
Initialise parser class.
This sets up the class defaults.
Options:
general=general: this over-rides and defaults with values
set in parser general can be of the form:
1. class ParamStorage (i.e. the same form
as self.general)
2. a command line list (where the first
item in the list is ignored
3. a string containing a set of command line general
See self.parse() for more details on general
2 and 3 as these simply make a call to tha
method.
log=True If log is set to True, then logging starts
when this class is instanced.
Note that the logfile and logdir might
change if subsequent calls to Parser.parse()
are made
"""
if type(args) == str:
args = args.split()
self.dolog = log
self.log = log
self.name = args[0]
self.args = args[1:]
self.fullargs = args
self.store_fullargs = args
if name == None:
import time
thistime = str(time.time())
name = type(self).__name__
name = "%s.%s" % (name, thistime)
self.thisname = name
# find the following flags:
# --conf | -c : conf
# --datadir : datadir
datadir = [".","~/.eoldas",sys.path[0]+'/../bin',sys.path[0]+'/../confs',\
sys.path[0]+'/../system_confs',sys.path[0]+'/../eoldaslib']
conf = "default.conf"
logfile = None
logdir = "."
self.top = ParamStorage()
self.top.general = ParamStorage()
self.top.general.__helper__ = ParamStorage()
self.top.general.__default__ = ParamStorage()
self.top.general.__extras__ = ParamStorage()
self.top.general.conf = []
for i in xrange(len(self.args)):
theseargs = self.args[i].split('=')
if theseargs[0] == "--conf":
conf = theseargs[1]
self.top.general.conf.append(conf)
elif theseargs[0][0:2] == "-c":
if len(theseargs) > 2:
conf = theseargs[0][2:]
else:
conf = self.args[i + 1]
self.top.general.conf.append(conf)
elif theseargs[0] == "--datadir":
datadir1 = theseargs[1].replace('[','').\
replace(']','').split()
[datadir1.append(datadir[i]) for i in \
xrange(len(datadir))]
datadir = datadir1
elif theseargs[0] == "--logfile":
logfile = theseargs[1]
elif theseargs[0] == "--logdir":
logdir = theseargs[1]
elif theseargs[0] == "--outdir":
outdir = theseargs[1]
if self.top.general.conf == []:
self.top.general.conf = conf
if logfile == None:
logfile = conf.replace('conf', 'log')
self.top.general.here = os.getcwd()
self.top.general.datadir = datadir
self.top.general.logfile = logfile
self.top.general.logdir = logdir
self.top.general.outdir = outdir
# add here to datadir
# in addition to '.' to take account of the change of directory
self.top.general.datadir = self.__add_here_to_datadir(\
self.top.general.here,self.top.general.datadir)
self.top.general.datadir = self.__add_here_to_datadir(\
self.top.general.outdir,self.top.general.datadir)
# cd to where the output is to be
self.__cd(self.top.general.outdir)
# set up the default command line options
self.default_loader()
# update with anything passed here
if general and type(general) == ParamStorage: self.top.update(\
self.__unload(general),combine=True)
# read the conf files to get any cmd line options
self.logger = sortlog(self,self.top.general.logfile,logger,name=self.thisname,\
logdir=self.top.general.logdir)
self.config = ConfFile(self.top.general.conf,name=self.thisname+'.config',\
loaders=self.loaders,datadir=self.top.\
general.datadir,logger=self.logger,logdir=self.top.general.logdir,\
logfile=self.top.general.logfile)
if len(self.config.configs) == 0:
this = "Warning: Nothing doing ... you haven't set any configuration",\
self.config.storelog
try:
self.logger(this)
except:
print "Called with args:"
print "eoldas", self.args
pass
raise Exception(this)
# now loaders contains all of the defaults set here
# plus those from the config (config opver-rides defaults here)
self.loaders = self.config.loaders
# now convert loaders into parser information
self.parseLoader(self.loaders)
self.parse(self.fullargs)
if general and type(general) == ParamStorage:
self.top.update(self.__unload(general), combine=True)
if general and type(general) == str:
self.parse(general.split())
if general and type(general) == list:
self.parse(general)
# now update the info in self.config
for i in self.config.infos:
i.update(self.top, combine=True)
# so now all terms in self.config.infos
# contain information from the config file, updated by
# the cmd line
i.logger = self.logger
i.log()
# move the information up a level
self.infos = self.config.infos
self.configs = self.config.configs
self.config_log = self.config.storelog
#if getopdir:
# self.sortnames()
self.config.loglist(self.top)
#del(self.config.infos)
#del(self.config.configs)
#del(self.config)
self.__cd(self.top.general.here)
def __add_here_to_datadir(self, here, datadir):
from os import sep, curdir, pardir
if type(datadir) == str:
datadir = [datadir]
iadd = 0
for i in xrange(len(datadir)):
j = i + iadd
if datadir[j] == curdir or datadir[j] == pardir:
tmp = datadir[:j]
rest = datadir[j + 1:]
tmp.append("%s%s%s" % (here, sep, datadir[j]))
tmp.append(datadir[j])
iadd += 1
for k in xrange(len(rest)):
tmp.append(rest[k])
datadir = tmp
return datadir
def default_loader(self):
"""
Load up parser information for first pass
"""
self.loaders = []
self.top.general.here = os.getcwd()
self.loaders.append(["datadir",['.',self.top.general.here],\
"Specify where the data and or conf files are"])
self.loaders.append(["passer",False,\
"Pass over optimisation (i.e. report and plot the initial values)"])
self.loaders.append(["outdir",None,\
"Explicitly mspecify the results and processing output directory"])
self.loaders.append(["verbose", False, "Switch ON verbose mode", "v"])
self.loaders.append(["debug", False, optparse.SUPPRESS_HELP, "d"])
self.loaders.append(["conf","default.conf",\
"Specify configuration file. Set multiple files by using the flag multiple times.","c"])
self.loaders.append(["logdir","logs",\
"Subdirectory to put log file in"])
self.loaders.append(["logfile", "logfile.logs", "Log file name"])
def parseLoader(self, loaders):
"""
Utility to load a set of terms from the list loaders
into the ParamStorage general
If there are 3 terms in each loaders element, they refer to:
1. name
2. default value
3. helper text
If there is a fourth, it is associated with extras
(short parser option)
"""
general = ParamStorage()
general.__default__ = ParamStorage()
general.__extras__ = ParamStorage()
general.__helper__ = ParamStorage()
for this in loaders:
if len(this) > 1:
general[this[0]] = this[1]
general.__default__[this[0]] = this[1]
else:
general[this[0]] = None
general.__default__[this[0]] = None
if len(this) > 2:
general.__helper__[this[0]] = this[2]
else:
general.__helper__[this[0]] = optparse.SUPPRESS_HELP
if len(this) > 3:
general.__extras__[this[0]] = "%s" % this[3]
else:
general.__extras__[this[0]] = None
# make sure arrays arent numpy.ndarray
if type(general.__default__[this[0]]) == np.ndarray:
general.__default__[this[0]] = \
list(general.__default__[this[0]])
self.top.update(self.__unload(general), combine=True)
def __list_to_string__(self, thisstr):
"""
Utility to convert a list to some useable string
"""
return(str(thisstr).replace('[','_').strip("']").\
replace('.dat','').replace("_'","_").replace(",","").\
replace(" ","").replace("''","_").replace("___","_").\
replace("__","_"))
def __cd(self, outdir):
if not os.path.exists(outdir):
try:
os.makedirs(outdir)
except OSerror:
print "Fatal: Prevented from creating", outdir
sys.exit(-1)
try:
os.chdir(outdir)
except:
print "Fatal: unable to cd to", outdir
raise Exception("Fatal: unable to cd to %s" % outdir)
def sortnames(self):
"""
Utility code to sort out some useful filenames & directories
"""
if self.top.general.outdir == None:
basename = self.top.general.basename
confnames = self.__list_to_string__(self.top.general.conf)
self.top.general.outdir = basename + "_conf_" + confnames
self.__cd(self.top.general.outdir)
def parse(self, args, log=False):
'''
Given a list such as sys.argv (of that form)
or an equivalent string parse the general and store
in self.parser
'''
self.dolog = log or self.dolog
if type(args) == type(""):
args = args.split()
args = args[1:]
self.top.general.cmdline = str(args)
usage = "usage: %prog [general] arg1 arg2"
parser = OptionParser(usage, version="%prog 0.1")
# we go through items in self.top.general and set up
# parser general for each
for this in sorted(self.top.general.__helper__.__dict__.keys()):
# sort out the action
# based on type of the default
default = self.top.general.__default__[this]
action = "store"
thistype = type(default)
if thistype == type(None):
thistype = type("")
argss = '--%s' % this
dest = "%s" % this
helper = self.top.general.__helper__[this]
if type(default) == type([]):
# list, so append
action = "store"
elif type(default) == type(True):
action = "store_true"
typer = "string"
if thistype != type([]) and thistype != type(True):
typer = '%s' % str(thistype).split("'")[1]
# has it got extras?
if self.top.general.__extras__[this] != None:
parser.add_option('-%s'%self.top.general.__extras__[\
this].lower(),argss,type="string",action=action,\
help=helper,default=str(default))
else:
parser.add_option(argss,action=action,help=helper,\
type="string",default=str(default))
elif (thistype != type(True)):
if self.top.general.__extras__[this] != None:
parser.add_option('-%s'%self.top.general.__extras__[\
this].lower(),argss,type="string",action=action,\
help=helper,default=str(default))
else:
parser.add_option(argss,action=action,help=helper,\
default=str(default))
if thistype == type(True):
if self.top.general.__extras__[this] != None:
parser.add_option('-%s'%self.top.general.__extras__[\
this].lower(),argss,dest=dest,action=action,\
help=helper,default=default)
else:
parser.add_option(argss,action=action,help=helper,\
dest=dest,default=default)
that = this.split('.')
argss = '--'
for i in xrange(len(that) - 1):
argss = argss + "%s." % that[i]
argss = argss + 'no_%s' % that[-1]
helper = 'The opposite of --%s' % this
action = 'store_false'
typer = '%s' % str(thistype).split("'")[1]
# has it got extras?
if self.top.general.__extras__[this] != None:
parser.add_option('-%s'%self.top.general.__extras__[\
this].capitalize(),argss,dest=dest,action=action,\
help=helper)
else:
parser.add_option(argss,action=action,dest=dest,\
help=helper)
# we have set all option types as str, so we need to interpret
# them in__unload
(general, args) = parser.parse_args(args)
#for data_file in args:
# general.data_file.append(data_file)
#general.data_file = list(np.array(general.data_file).flatten())
#general.brf= list(np.array(general.brf).flatten())
# load these into self.general
self.top.update(self.__unload(general.__dict__), combine=True)
#self.sortnames()
if self.dolog:
self.log = set_up_logfile(self.top.general.logfile,\
logdir=self.top.general.logdir)
self.log_report()
def __unload(self, options):
from eoldas_ConfFile import array_type_convert
this = ParamStorage()
this.general = ParamStorage()
for (k, v) in options.iteritems():
ps = this
that = k.split('.')
if len(that) == 1:
ps = this.general
else:
for i in xrange(len(that) - 1):
if not hasattr(ps, that[i]):
ps[that[i]] = ParamStorage()
ps = ps[that[i]]
# set the value v which needs to
# to be interpreted
ps[that[-1]] = array_type_convert(self.top, v)
return this
