class SpecialVariable(ParamStorage): ''' A class that can deal with the datatypes needed for eoldas It allows a variable to be set to various data types and interprets these into the data structure. The data structure imposed here is: self.data.this self.name.this to store some item called 'this'. Other information can be stored as well but part of the idea here is to have some imposed constraints on the data structure so that we can sensibly load up odfferent datasets. The idea is that data will be stored in self.data and associated metadata in self.name. If items are given the same name in both sub-structures, we can easily keep track of them. There is no actual requirement that this is adhered to, but it is certainy permitted and encouraged for the indended use of this class. Probably the most important thing then about this class is that is a SpecialVariable is assigned different data types, then it can do sensible things with them in the context of the EOLDAS (and wider applications). When an assignment takes place (either of ther form self.state = foo or self['state'] = foo then what is actually stored depends on the type and nature of foo. The main features as follows: If foo is a string: A guess is made that it is a filename, and an attempt is made to read the file. All directories in the list self.dirnames are searched for the filename, and any readable files found are considered candidates, Each of these is read in turn. A set of potential data formats, specified by the readers in readers (self.reader_functions) is considered, as if a sucessful interpretation takes place the data is returned and stiored in the derired variable. So, for example, if we have self.state = foo as above and foo is a valid, readable file in the list of directories specified, and it is interprable with one of the formats defined, then the main dataset is loaded into: self.data.state (alternatively known as self.data['state']). If foo is a ParamStorage, it should have the same structure as that here (i.e. self.data and self.name) and these structures are then loaded. If foo is a dictionary (type dict) It is first converted to a ParamStorage and then loaded as above. If foo is any other datatype, it is left pretty much as it, except that an attempt to convert to a np.array is made. Depending on the format, there might be data other than the main dataset (e.g. locational information) and these are loaded by the loaders into relevant oarts of self.data and self.name. For classes that use this class for EOLDAS, we will typically use: self.data.state : state variable data self.data.sd : uncertainty information as sd (or a similar fuller representation) self.data.control : control information (e.g. view angles) self.data.location : location information with associated descriptor data in the relevant parts of self.name. The idea for simple use of the data structure then is for all of these datasets represented as 2D datasets, where the number of rows in each of the self.data.state etc field will be the same, but the number of columns will tend to vary (e.g different numbers of state variables). The reason for considering such a 2D 'flat'(ish) representation is that it is easy to tabulate and understand. In fact the data will be of quite high dimension. E.g. is the data vary by time, x and y, then we would have 3 columns for self.data.location, with descriptors for the columns in self.name.location, and corresponding state data in self.data.state (with the number of state variables determining the number of columns in that table). As mentioned, at the pooint of this class, there is no strict requirement for any such structure ti the data loaded or used, but that is the plan for EOLDAS use, so worth documenting at this point. ''' def __init__(self,info=[],name=None,thisname=None,readers=[],log_terms={},\ datadir=["."],env=None,\ header=None,writers={}, simple=False,logger=None): ''' Class SpecialVariable initialisation. Sets up the class as a ParamStorage and calls self.init() See init() fort a fuller descripotion of the options. ''' ParamStorage.__init__(self,logger=logger) name = name or thisname if name == None: import time thistime = str(time.time()) name = type(self).__name__ name = "%s.%s" % (name,thistime) self.thisname = name self.init(info=[],name=self.thisname,readers=readers,log_terms={},\ datadir=datadir,env=env,\ header=header,writers=writers,\ simple=False,logger=logger) def init(self,info=[],name=None,readers=[],log_terms={},\ datadir=None,env=None,\ header=None,writers={},\ simple=False,logger=None): ''' Initialise information in a SpecialVariable instance. Options: info : Information tthat can be passed through to reader methods (a list). thisname : a name to use to identify this instance in any logging. By default this is None. If thisname is set to True, then logging is to stdout. readers : A list of reader methods that are pre-pended to those already contained in the class. log_terms : A dictionary of log options. By default {'logfile':None,'logdir':'log','debug':True} If thisname is set, and logfile specified, then logs are logged to that file. If thisname is set to True, then logging is to stdout. datadir : A list of directories to search for data files to interpret if the SpecialVariable is set to a string. env : An environment variable that can be used to extend the datadir variable. header : A header string to use to identify pickle files. By default, this is set to "EOLDAS -- plewis -- UCL -- V0.1" simple : A flag to swicth off the 'complicated' interpretation methods, i.e. just set and return variables literally, do not try to interpret them. ''' self.set('simple',True) if name == None: import time thistime = str(time.time()) name = type(self).__name__ name = "%s.%s" % (name,thistime) self.thisname = name # this is where we will put any data self.data = ParamStorage() self.name = ParamStorage() self.info = info self.datadir = datadir or ['.'] self.env = env init_read_write(self,header,readers,writers) # sort logging and log if thisname != None self.log_terms = {'logfile':None,'logdir':'log','debug':True} # override logging info for (key,value) in log_terms.iteritems(): self.log_terms[key] = value self.logger= sortlog(self,self.log_terms['logfile'],logger,name=self.thisname,\ logdir=self.log_terms['logdir'],\ debug=self.log_terms['debug']) self.simple = simple set = lambda self,this,value :ParamStorage.__setattr__(self,this,value) set.__name__ = 'set' set.__doc__ = """ A method to set the literal value of this, rather than attempt an interpretation (e.g. used when self.simple is True) """ get = lambda self,this :ParamStorage.__getattr__(self,this) get.__name__ = 'get' get.__doc__ = """ A method to get the literal value of this, rather than attempt an interpretation (e.g. used when self.simple is True) """ def __setitem__(self,this,value): ''' Variable setting method for style self['this']. Interpreted the same as via __setattr__. ''' # always set the item self.__setattr__(this,value) def __setattr__(self,this,value): ''' Variable setting method for style self.this Varies what it does depending on the type of value. The method interprets and sets the SpecialVariable value: 1. ParamStorage or SpecialVariable. The data are directly loaded. This is one of the most flexible formats for input. It expects fields 'data' and/or 'name', which are loaded into self. There will normally be a field data.this, where this is the variable name passed here. 2. A dictionary, same format as the ParamStorage. 3. A tuple, interpreted as (data,name) and loaded accordingly. 4. *string* as filename (various formats). An attempt to read the string as a file (of a set of formats) is made. If none pass then it it maintained as a string. 5. A numpy array (np.array) that is loaded into self.data.this. 6. Anything else. Loaded into self.data.this as a numpy array. ''' if self.simple: self.set(this,value) return t = type(value) try: if t == ParamStorage or t == SpecialVariable: # update the whole structure #self.__set_if_unset('data',ParamStorage()) #self.__set_if_unset('name',ParamStorage()) self.data.update(value.data,combine=True) self.name.update(value.name,combine=True) elif t == dict: n_value = ParamStorage().from_dict(value) self.__setattr__(this,n_value) elif t == tuple or t == list: # assumed to be (data,name) or [data,name] #self.__set_if_unset('data',ParamStorage()) #self.__set_if_unset('name',ParamStorage()) #ParamStorage.__setattr__(self['data'],this,value[0]) #ParamStorage.__setattr__(self['name'],this,value[1]) ParamStorage.__setattr__(self['data'],this,np.array(value)) elif t == str: # set the term #self.__set_if_unset('data',ParamStorage()) #self.__set_if_unset('name',ParamStorage()) ParamStorage.__setattr__(self['data'],this,value) # interpret as a file read if possible self.process_data_string(this,info=self.info) elif t == np.ndarray: #self.__set_if_unset('data',ParamStorage()) #self.__set_if_unset('name',ParamStorage()) ParamStorage.__setattr__(self['data'],this,value) else: ParamStorage.__setattr__(self['data'],this,\ np.array(value)) except: if self.logger: self.logger.info("Failed to set SpecialVariable %s from %s %s"\ %(this,t.__name__,value)) return if self.logger: self.logger.info("Set variable %s from type %s"%(this,t.__name__)) def __getattr__(self,name): ''' Variable getting method for style self.this If the field 'data' exists in self.__dict__ and 'name' is in the dictionary, then the field self.data.this is returned. Otherwise, if the field 'name' is in self.__dict__, self.name is returned. Otherwise return None. ''' if 'data' in self.__dict__ and name in self.data.__dict__: return self.data.__dict__.__getitem__ ( name ) elif name in self.__dict__: return self.__dict__.__getitem__ ( name ) else: return None def __getitem__(self,name): ''' Variable getting method for style self['this']. Interpreted the same as via __getattr__. ''' # first look in data if 'data' in self.__dict__ and name in self.data.__dict__: return self.data.__dict__.__getitem__ ( name ) elif name in self.__dict__: return self.__dict__.__getitem__ ( name ) else: return None def process_data_string(self,name,info=[],fmt=None): ''' Attempt to load data from a string, assuming the string is a filename. The array self.datadir is searched for readable files with the string 'name' (also self.env), and a list of potential files considered for reading. Each readable file is passed to self.read, and if it is interpretable, it is loaded according to the read method. Note tha the format can be specified. If not, then all formats are attempted until a sucessful read is made. ''' from eoldas_Lib import get_filename orig = self.data[name] if self.logger: self.logger.debug('%s is a string ... see if its a readable file ...' \ % name) # find a list of potential files goodfiles, is_error = get_filename(orig,datadir=self.datadir,\ env=self.env,multiple=True) if is_error[0] and self.logger: self.logger.debug(str(is_error[1])) return if self.logger: self.logger.debug("*** looking at potential files %s"%str(goodfiles)) # loop over all files that it might be for goodfile in goodfiles: stuff,is_error = reader(self,goodfile,name,fmt=fmt,info=info) if not is_error[0] and self.logger: self.logger.info("Read file %s "%goodfile) return if self.logger: self.logger.debug(self.error_msg) return write = lambda self,filename,fmt : writer(self,filename,None,fmt=fmt) read = lambda self,filename,fmt : reader(self,filename,None,fmt=fmt,info=[])
class Parser(): def __init__(self,args,name=None,general=None,log=False,logger=None,outdir=".",getopdir=False,parse=True): """ Initialise parser class. This sets up the class defaults. Options: general=general: this over-rides and defaults with values set in parser general can be of the form: 1. class ParamStorage (i.e. the same form as self.general) 2. a command line list (where the first item in the list is ignored 3. a string containing a set of command line general See self.parse() for more details on general 2 and 3 as these simply make a call to tha method. log=True If log is set to True, then logging starts when this class is instanced. Note that the logfile and logdir might change if subsequent calls to Parser.parse() are made """ if type(args) == str: args = args.split() self.dolog = log self.log = log self.name = args[0] self.args = args[1:] self.fullargs = args self.store_fullargs = args if name == None: import time thistime = str(time.time()) name = type(self).__name__ name = "%s.%s" % (name,thistime) self.thisname = name # find the following flags: # --conf | -c : conf # --datadir : datadir datadir = [".","~/.eoldas",sys.path[0]+'/../bin',sys.path[0]+'/../confs',\ sys.path[0]+'/../system_confs',sys.path[0]+'/../eoldaslib'] conf = "default.conf" logfile = None logdir = "." self.top = ParamStorage () self.top.general = ParamStorage () self.top.general.__helper__ = ParamStorage () self.top.general.__default__ = ParamStorage () self.top.general.__extras__ = ParamStorage () self.top.general.conf = [] for i in xrange(len(self.args)): theseargs = self.args[i].split('=') if theseargs[0] == "--conf": conf = theseargs[1] self.top.general.conf.append(conf) elif theseargs[0][0:2] == "-c": if len(theseargs) > 2: conf = theseargs[0][2:] else: conf = self.args[i+1] self.top.general.conf.append(conf) elif theseargs[0] == "--datadir": datadir1 = theseargs[1].replace('[','').\ replace(']','').split() [datadir1.append(datadir[i]) for i in \ xrange(len(datadir))] datadir = datadir1 elif theseargs[0] == "--logfile": logfile= theseargs[1] elif theseargs[0] == "--logdir": logdir = theseargs[1] elif theseargs[0] == "--outdir": outdir = theseargs[1] if self.top.general.conf == []: self.top.general.conf = conf if logfile == None: logfile = conf.replace('conf','log') self.top.general.here = os.getcwd() self.top.general.datadir = datadir self.top.general.logfile = logfile self.top.general.logdir = logdir self.top.general.outdir = outdir # add here to datadir # in addition to '.' to take account of the change of directory self.top.general.datadir = self.__add_here_to_datadir(\ self.top.general.here,self.top.general.datadir) self.top.general.datadir = self.__add_here_to_datadir(\ self.top.general.outdir,self.top.general.datadir) # cd to where the output is to be self.__cd(self.top.general.outdir) # set up the default command line options self.default_loader() # update with anything passed here if general and type(general) == ParamStorage: self.top.update(\ self.__unload(general),combine=True) # read the conf files to get any cmd line options self.logger = sortlog(self,self.top.general.logfile,logger,name=self.thisname,\ logdir=self.top.general.logdir) self.config = ConfFile(self.top.general.conf,name=self.thisname+'.config',\ loaders=self.loaders,datadir=self.top.\ general.datadir,logger=self.logger,logdir=self.top.general.logdir,\ logfile=self.top.general.logfile) if len(self.config.configs) == 0: this = "Warning: Nothing doing ... you haven't set any configuration",\ self.config.storelog try: self.logger(this) except: print "Called with args:" print "eoldas",self.args pass raise Exception(this) # now loaders contains all of the defaults set here # plus those from the config (config opver-rides defaults here) self.loaders = self.config.loaders # now convert loaders into parser information self.parseLoader(self.loaders) self.parse(self.fullargs) if general and type(general) == ParamStorage: self.top.update(self.__unload(general),combine=True) if general and type(general) == str: self.parse(general.split()) if general and type(general) == list: self.parse(general) # now update the info in self.config for i in self.config.infos: i.update(self.top,combine=True) # so now all terms in self.config.infos # contain information from the config file, updated by # the cmd line i.logger = self.logger i.log() # move the information up a level self.infos = self.config.infos self.configs = self.config.configs self.config_log = self.config.storelog #if getopdir: # self.sortnames() self.config.loglist(self.top) #del(self.config.infos) #del(self.config.configs) #del(self.config) self.__cd(self.top.general.here) def __add_here_to_datadir(self,here,datadir): from os import sep,curdir,pardir if type(datadir) == str: datadir = [datadir] iadd = 0 for i in xrange(len(datadir)): j = i + iadd if datadir[j] == curdir or datadir[j] == pardir: tmp = datadir[:j] rest = datadir[j+1:] tmp.append("%s%s%s" % (here,sep,datadir[j])) tmp.append(datadir[j]) iadd += 1 for k in xrange(len(rest)): tmp.append(rest[k]) datadir = tmp return datadir def default_loader(self): """ Load up parser information for first pass """ self.loaders = [] self.top.general.here = os.getcwd() self.loaders.append(["datadir",['.',self.top.general.here],\ "Specify where the data and or conf files are"]) self.loaders.append(["passer",False,\ "Pass over optimisation (i.e. report and plot the initial values)"]) self.loaders.append(["outdir",None,\ "Explicitly mspecify the results and processing output directory"]) self.loaders.append(["verbose",False,"Switch ON verbose mode","v"]) self.loaders.append(["debug",False,optparse.SUPPRESS_HELP,"d"]) self.loaders.append(["conf","default.conf",\ "Specify configuration file. Set multiple files by using the flag multiple times.","c"]) self.loaders.append(["logdir","logs",\ "Subdirectory to put log file in"]) self.loaders.append(["logfile","logfile.logs","Log file name"]) def parseLoader(self,loaders): """ Utility to load a set of terms from the list loaders into the ParamStorage general If there are 3 terms in each loaders element, they refer to: 1. name 2. default value 3. helper text If there is a fourth, it is associated with extras (short parser option) """ general = ParamStorage () general.__default__ = ParamStorage () general.__extras__ = ParamStorage () general.__helper__ = ParamStorage () for this in loaders: if len(this) > 1: general[this[0]] = this[1] general.__default__[this[0]] = this[1] else: general[this[0]] = None general.__default__[this[0]] = None if len(this) > 2: general.__helper__[this[0]] = this[2] else: general.__helper__[this[0]] = optparse.SUPPRESS_HELP if len(this) > 3: general.__extras__[this[0]] = "%s" % this[3] else: general.__extras__[this[0]] = None # make sure arrays arent numpy.ndarray if type(general.__default__[this[0]]) == np.ndarray: general.__default__[this[0]] = \ list(general.__default__[this[0]]) self.top.update(self.__unload(general),combine=True) def __list_to_string__(self,thisstr): """ Utility to convert a list to some useable string """ return(str(thisstr).replace('[','_').strip("']").\ replace('.dat','').replace("_'","_").replace(",","").\ replace(" ","").replace("''","_").replace("___","_").\ replace("__","_")); def __cd(self,outdir): if not os.path.exists(outdir): try: os.makedirs(outdir) except OSerror: print "Fatal: Prevented from creating",outdir sys.exit(-1) try: os.chdir(outdir) except: print "Fatal: unable to cd to",outdir raise Exception("Fatal: unable to cd to %s"%outdir) def sortnames(self): """ Utility code to sort out some useful filenames & directories """ if self.top.general.outdir == None: basename = self.top.general.basename confnames = self.__list_to_string__(self.top.general.conf) self.top.general.outdir = basename + "_conf_" + confnames self.__cd(self.top.general.outdir) def parse(self,args,log=False): ''' Given a list such as sys.argv (of that form) or an equivalent string parse the general and store in self.parser ''' self.dolog = log or self.dolog if type(args) == type(""): args = args.split() args = args[1:] self.top.general.cmdline = str(args) usage = "usage: %prog [general] arg1 arg2" parser = OptionParser(usage,version="%prog 0.1") # we go through items in self.top.general and set up # parser general for each for this in sorted(self.top.general.__helper__.__dict__.keys()): # sort out the action # based on type of the default default=self.top.general.__default__[this] action="store" thistype=type(default) if thistype == type(None): thistype = type("") argss = '--%s'%this dest = "%s"%this helper = self.top.general.__helper__[this] if type(default) == type([]): # list, so append action="store" elif type(default) == type(True): action="store_true" typer = "string" if thistype != type([]) and thistype != type(True): typer='%s' % str(thistype).split("'")[1] # has it got extras? if self.top.general.__extras__[this] != None: parser.add_option('-%s'%self.top.general.__extras__[\ this].lower(),argss,type="string",action=action,\ help=helper,default=str(default)) else: parser.add_option(argss,action=action,help=helper,\ type="string",default=str(default)) elif ( thistype != type(True)): if self.top.general.__extras__[this] != None: parser.add_option('-%s'%self.top.general.__extras__[\ this].lower(),argss,type="string",action=action,\ help=helper,default=str(default)) else: parser.add_option(argss,action=action,help=helper,\ default=str(default)) if thistype == type(True): if self.top.general.__extras__[this] != None: parser.add_option('-%s'%self.top.general.__extras__[\ this].lower(),argss,dest=dest,action=action,\ help=helper,default=default) else: parser.add_option(argss,action=action,help=helper,\ dest=dest,default=default) that = this.split('.') argss = '--' for i in xrange(len(that)-1): argss = argss + "%s." % that[i] argss = argss + 'no_%s' % that[-1] helper='The opposite of --%s'%this action='store_false' typer='%s' % str(thistype).split("'")[1] # has it got extras? if self.top.general.__extras__[this] != None: parser.add_option('-%s'%self.top.general.__extras__[\ this].capitalize(),argss,dest=dest,action=action,\ help=helper) else: parser.add_option(argss,action=action,dest=dest,\ help=helper) # we have set all option types as str, so we need to interpret # them in__unload (general, args) = parser.parse_args(args) #for data_file in args: # general.data_file.append(data_file) #general.data_file = list(np.array(general.data_file).flatten()) #general.brf= list(np.array(general.brf).flatten()) # load these into self.general self.top.update(self.__unload(general.__dict__),combine=True) #self.sortnames() if self.dolog: self.log = set_up_logfile(self.top.general.logfile,\ logdir=self.top.general.logdir) self.log_report() def __unload(self,options): from eoldas_ConfFile import array_type_convert this = ParamStorage() this.general = ParamStorage() for (k,v) in options.iteritems(): ps = this that = k.split('.') if len(that) == 1: ps = this.general else: for i in xrange(len(that)-1): if not hasattr(ps,that[i]): ps[that[i]] = ParamStorage() ps = ps[that[i]] # set the value v which needs to # to be interpreted ps[that[-1]] = array_type_convert(self.top,v) return this
class Parser(): def __init__(self, args, name=None, general=None, log=False, logger=None, outdir=".", getopdir=False, parse=True): """ Initialise parser class. This sets up the class defaults. Options: general=general: this over-rides and defaults with values set in parser general can be of the form: 1. class ParamStorage (i.e. the same form as self.general) 2. a command line list (where the first item in the list is ignored 3. a string containing a set of command line general See self.parse() for more details on general 2 and 3 as these simply make a call to tha method. log=True If log is set to True, then logging starts when this class is instanced. Note that the logfile and logdir might change if subsequent calls to Parser.parse() are made """ if type(args) == str: args = args.split() self.dolog = log self.log = log self.name = args[0] self.args = args[1:] self.fullargs = args self.store_fullargs = args if name == None: import time thistime = str(time.time()) name = type(self).__name__ name = "%s.%s" % (name, thistime) self.thisname = name # find the following flags: # --conf | -c : conf # --datadir : datadir datadir = [".","~/.eoldas",sys.path[0]+'/../bin',sys.path[0]+'/../confs',\ sys.path[0]+'/../system_confs',sys.path[0]+'/../eoldaslib'] conf = "default.conf" logfile = None logdir = "." self.top = ParamStorage() self.top.general = ParamStorage() self.top.general.__helper__ = ParamStorage() self.top.general.__default__ = ParamStorage() self.top.general.__extras__ = ParamStorage() self.top.general.conf = [] for i in xrange(len(self.args)): theseargs = self.args[i].split('=') if theseargs[0] == "--conf": conf = theseargs[1] self.top.general.conf.append(conf) elif theseargs[0][0:2] == "-c": if len(theseargs) > 2: conf = theseargs[0][2:] else: conf = self.args[i + 1] self.top.general.conf.append(conf) elif theseargs[0] == "--datadir": datadir1 = theseargs[1].replace('[','').\ replace(']','').split() [datadir1.append(datadir[i]) for i in \ xrange(len(datadir))] datadir = datadir1 elif theseargs[0] == "--logfile": logfile = theseargs[1] elif theseargs[0] == "--logdir": logdir = theseargs[1] elif theseargs[0] == "--outdir": outdir = theseargs[1] if self.top.general.conf == []: self.top.general.conf = conf if logfile == None: logfile = conf.replace('conf', 'log') self.top.general.here = os.getcwd() self.top.general.datadir = datadir self.top.general.logfile = logfile self.top.general.logdir = logdir self.top.general.outdir = outdir # add here to datadir # in addition to '.' to take account of the change of directory self.top.general.datadir = self.__add_here_to_datadir(\ self.top.general.here,self.top.general.datadir) self.top.general.datadir = self.__add_here_to_datadir(\ self.top.general.outdir,self.top.general.datadir) # cd to where the output is to be self.__cd(self.top.general.outdir) # set up the default command line options self.default_loader() # update with anything passed here if general and type(general) == ParamStorage: self.top.update(\ self.__unload(general),combine=True) # read the conf files to get any cmd line options self.logger = sortlog(self,self.top.general.logfile,logger,name=self.thisname,\ logdir=self.top.general.logdir) self.config = ConfFile(self.top.general.conf,name=self.thisname+'.config',\ loaders=self.loaders,datadir=self.top.\ general.datadir,logger=self.logger,logdir=self.top.general.logdir,\ logfile=self.top.general.logfile) if len(self.config.configs) == 0: this = "Warning: Nothing doing ... you haven't set any configuration",\ self.config.storelog try: self.logger(this) except: print "Called with args:" print "eoldas", self.args pass raise Exception(this) # now loaders contains all of the defaults set here # plus those from the config (config opver-rides defaults here) self.loaders = self.config.loaders # now convert loaders into parser information self.parseLoader(self.loaders) self.parse(self.fullargs) if general and type(general) == ParamStorage: self.top.update(self.__unload(general), combine=True) if general and type(general) == str: self.parse(general.split()) if general and type(general) == list: self.parse(general) # now update the info in self.config for i in self.config.infos: i.update(self.top, combine=True) # so now all terms in self.config.infos # contain information from the config file, updated by # the cmd line i.logger = self.logger i.log() # move the information up a level self.infos = self.config.infos self.configs = self.config.configs self.config_log = self.config.storelog #if getopdir: # self.sortnames() self.config.loglist(self.top) #del(self.config.infos) #del(self.config.configs) #del(self.config) self.__cd(self.top.general.here) def __add_here_to_datadir(self, here, datadir): from os import sep, curdir, pardir if type(datadir) == str: datadir = [datadir] iadd = 0 for i in xrange(len(datadir)): j = i + iadd if datadir[j] == curdir or datadir[j] == pardir: tmp = datadir[:j] rest = datadir[j + 1:] tmp.append("%s%s%s" % (here, sep, datadir[j])) tmp.append(datadir[j]) iadd += 1 for k in xrange(len(rest)): tmp.append(rest[k]) datadir = tmp return datadir def default_loader(self): """ Load up parser information for first pass """ self.loaders = [] self.top.general.here = os.getcwd() self.loaders.append(["datadir",['.',self.top.general.here],\ "Specify where the data and or conf files are"]) self.loaders.append(["passer",False,\ "Pass over optimisation (i.e. report and plot the initial values)"]) self.loaders.append(["outdir",None,\ "Explicitly mspecify the results and processing output directory"]) self.loaders.append(["verbose", False, "Switch ON verbose mode", "v"]) self.loaders.append(["debug", False, optparse.SUPPRESS_HELP, "d"]) self.loaders.append(["conf","default.conf",\ "Specify configuration file. Set multiple files by using the flag multiple times.","c"]) self.loaders.append(["logdir","logs",\ "Subdirectory to put log file in"]) self.loaders.append(["logfile", "logfile.logs", "Log file name"]) def parseLoader(self, loaders): """ Utility to load a set of terms from the list loaders into the ParamStorage general If there are 3 terms in each loaders element, they refer to: 1. name 2. default value 3. helper text If there is a fourth, it is associated with extras (short parser option) """ general = ParamStorage() general.__default__ = ParamStorage() general.__extras__ = ParamStorage() general.__helper__ = ParamStorage() for this in loaders: if len(this) > 1: general[this[0]] = this[1] general.__default__[this[0]] = this[1] else: general[this[0]] = None general.__default__[this[0]] = None if len(this) > 2: general.__helper__[this[0]] = this[2] else: general.__helper__[this[0]] = optparse.SUPPRESS_HELP if len(this) > 3: general.__extras__[this[0]] = "%s" % this[3] else: general.__extras__[this[0]] = None # make sure arrays arent numpy.ndarray if type(general.__default__[this[0]]) == np.ndarray: general.__default__[this[0]] = \ list(general.__default__[this[0]]) self.top.update(self.__unload(general), combine=True) def __list_to_string__(self, thisstr): """ Utility to convert a list to some useable string """ return(str(thisstr).replace('[','_').strip("']").\ replace('.dat','').replace("_'","_").replace(",","").\ replace(" ","").replace("''","_").replace("___","_").\ replace("__","_")) def __cd(self, outdir): if not os.path.exists(outdir): try: os.makedirs(outdir) except OSerror: print "Fatal: Prevented from creating", outdir sys.exit(-1) try: os.chdir(outdir) except: print "Fatal: unable to cd to", outdir raise Exception("Fatal: unable to cd to %s" % outdir) def sortnames(self): """ Utility code to sort out some useful filenames & directories """ if self.top.general.outdir == None: basename = self.top.general.basename confnames = self.__list_to_string__(self.top.general.conf) self.top.general.outdir = basename + "_conf_" + confnames self.__cd(self.top.general.outdir) def parse(self, args, log=False): ''' Given a list such as sys.argv (of that form) or an equivalent string parse the general and store in self.parser ''' self.dolog = log or self.dolog if type(args) == type(""): args = args.split() args = args[1:] self.top.general.cmdline = str(args) usage = "usage: %prog [general] arg1 arg2" parser = OptionParser(usage, version="%prog 0.1") # we go through items in self.top.general and set up # parser general for each for this in sorted(self.top.general.__helper__.__dict__.keys()): # sort out the action # based on type of the default default = self.top.general.__default__[this] action = "store" thistype = type(default) if thistype == type(None): thistype = type("") argss = '--%s' % this dest = "%s" % this helper = self.top.general.__helper__[this] if type(default) == type([]): # list, so append action = "store" elif type(default) == type(True): action = "store_true" typer = "string" if thistype != type([]) and thistype != type(True): typer = '%s' % str(thistype).split("'")[1] # has it got extras? if self.top.general.__extras__[this] != None: parser.add_option('-%s'%self.top.general.__extras__[\ this].lower(),argss,type="string",action=action,\ help=helper,default=str(default)) else: parser.add_option(argss,action=action,help=helper,\ type="string",default=str(default)) elif (thistype != type(True)): if self.top.general.__extras__[this] != None: parser.add_option('-%s'%self.top.general.__extras__[\ this].lower(),argss,type="string",action=action,\ help=helper,default=str(default)) else: parser.add_option(argss,action=action,help=helper,\ default=str(default)) if thistype == type(True): if self.top.general.__extras__[this] != None: parser.add_option('-%s'%self.top.general.__extras__[\ this].lower(),argss,dest=dest,action=action,\ help=helper,default=default) else: parser.add_option(argss,action=action,help=helper,\ dest=dest,default=default) that = this.split('.') argss = '--' for i in xrange(len(that) - 1): argss = argss + "%s." % that[i] argss = argss + 'no_%s' % that[-1] helper = 'The opposite of --%s' % this action = 'store_false' typer = '%s' % str(thistype).split("'")[1] # has it got extras? if self.top.general.__extras__[this] != None: parser.add_option('-%s'%self.top.general.__extras__[\ this].capitalize(),argss,dest=dest,action=action,\ help=helper) else: parser.add_option(argss,action=action,dest=dest,\ help=helper) # we have set all option types as str, so we need to interpret # them in__unload (general, args) = parser.parse_args(args) #for data_file in args: # general.data_file.append(data_file) #general.data_file = list(np.array(general.data_file).flatten()) #general.brf= list(np.array(general.brf).flatten()) # load these into self.general self.top.update(self.__unload(general.__dict__), combine=True) #self.sortnames() if self.dolog: self.log = set_up_logfile(self.top.general.logfile,\ logdir=self.top.general.logdir) self.log_report() def __unload(self, options): from eoldas_ConfFile import array_type_convert this = ParamStorage() this.general = ParamStorage() for (k, v) in options.iteritems(): ps = this that = k.split('.') if len(that) == 1: ps = this.general else: for i in xrange(len(that) - 1): if not hasattr(ps, that[i]): ps[that[i]] = ParamStorage() ps = ps[that[i]] # set the value v which needs to # to be interpreted ps[that[-1]] = array_type_convert(self.top, v) return this