def initialize( ): """ initialize SLIMpy environment """ env = InstanceManager() from slimpy_base.Environment.hashTable import _HashTable from slimpy_base.Environment.GlobalVars import GlobalVars as _GlobalVars from slimpy_base.Environment.GraphManager import GraphManager from slimpy_base.Environment.KeyStone import KeyStone from slimpy_base.Core.MutiProcessUtils.EmploymentCenter import EmploymentCenter from slimpy_base.utils.Records import Records #singleton class any and all instances are the same in the same environment env['graphmgr'] = GraphManager # order matters fifo for exit funcs env['table'] = _HashTable env['record'] = Records env['slimvars'] = _GlobalVars env['keystone'] = KeyStone env['center'] = EmploymentCenter env.assure_new_instances( )
def slimpy_variable_emitter(target, source, env): """ add dependencies on SLIMpy global variables if a variable defined in slimvars and specified in env, then slimpy adds it as a Value source """ from slimpy_base.Environment.InstanceManager import InstanceManager slimvars = InstanceManager()['slimvars'] for key in slimvars.keys(): if env.has_key(key): par = env[key] val = env.Value((key, par)) source.append(val) return target, source
def windowm_mpi_runtime_map(command): _env = InstanceManager() nodemap = _env['slimvars'].get_node_map() params = [] nodes = [node for node, _ in nodemap] files = command.params dist = even_dist(nodes, files, 0) for (container, node_dist) in izip(files, dist): if isinstance(node_dist, tuple): from_node, to_node = node_dist assert from_node in container.nodenames container.node_copy(to_node) else: from_node, to_node = node_dist, node_dist assert container.available_from(to_node) if to_node in ['localhost', '']: to_node = getfqdn() params.append("%s:%s" % (to_node, container.get_data(to_node))) return params, command.kparams
class SLIMBuilder(object): """ abstract class """ # log = Log() # table = HashTable() # slimvars = GlobalVars() env = InstanceManager() def isSource(self, val): """ test is val is a DataContainer or id of a DataContainer and if that DataContainer is full """ try: node = self.env['table'][val] except KeyError: return False if isinstance(node, DataContainer): return node.isfull() return False
def format(self, nodename, params, kparams): """ creates a dictionary with the key "cmnd" which is a string of all the """ env = InstanceManager() table = env['table'] l = lambda a, b: str(a) + '="' + str(b) + '"' cs = [self.getCmnd()] push = cs.append for par in params: if isinstance(par, Node): src = table[par.getID()] par = src.get_data(nodename) push(par) for key, val in kparams.items(): if isinstance(val, Node): val = val.get_data(nodename) push(l(key, val)) # p = " ".join(map(str, params)) # k = " ".join(map(l , kparams.items())) # cs = " ".join([self.getCmnd(),p, k]) return " ".join(map(str, cs))
def __new_instance__(self, name): ''' create a new instance ''' Singleton.__new_instance__(self, name) self.lock = RLock() with self: self._jobs_posted = 0 self._jobs_finished = 0 self._done = False self._error = False self._error_message = None self._pids = set() self._aborted_pids = set() self._waiting_list = set() self._event = Event() self.__node_names = [] self.nodes = {} self._idle = set() self._fin = set() self._head_node = ("__main__", "__main__") self.env = InstanceManager() self._semaphore = None
def testprepend_datapath(self): env = InstanceManager() local_dp = 'local_dp' tmp_global_dp = 'tmp_global_dp' global_dp = 'global_dp' env['slimvars']['localtmpdir'] = local_dp env['slimvars']['globaltmpdir'] = tmp_global_dp env['slimvars']['datapath'] = global_dp itr = [ [(None, None), None], [(False, None), None], [(True, None), None], [(None, True), None], [(True, True), local_dp], [(False, True), tmp_global_dp], [(None, False), global_dp], [(True, False), global_dp], [(False, False), global_dp], ] for (is_loc, is_tmp), ans in itr: res = Unix_Pipes.prepend_datapath(is_loc, is_tmp, 'com') if ans is None: self.assertFalse("DATAPATH" in res, "expected no datapath") else: self.assertTrue(ans in res, "expected %(ans) got %(res)s")
class Worker(Thread, Runner): """ Worker runs jobs """ env = InstanceManager() def __init__(self, name, master_id, processor=0): self.master_id = master_id self.processor = processor self.env['center'].subscribe(name, processor) Thread.__init__(self, name=name) self.pid = 0 self.thread_state = "init" def _get_ts(self): return self._thread_state def _set_ts(self, val): self._thread_state = val return # print self, val thread_state = property(_get_ts, _set_ts) ## print status of thread as a traceback def status(self): """ print status of thread """ if self.isAlive(): f = _current_frames()[self._ident] traceback.print_stack(f) else: print "Not Alive" def run(self): ''' run method needed by thread class calls self.main_loop ''' self.thread_state = "running" self._ident = get_ident() self.env.current_env = self.master_id try: self.main_loop() except Exception, msg: # print "worker calling error" self.env['center'].error = True, (str(self), str(msg)) self.safe_release_lock() # print self,'Thread Terminated' self.thread_state = "exit error" raise self.thread_state = "exit" return
class MultiCoreRunner(presStack): """ Runs commands from the graph. uses two stacks to keep track of dependencies and perhaps run several commands at the same time. """ env = InstanceManager() def __init__(self): self.created_nodes = set() self.nodes = None presStack.__init__(self) def set_graph(self, graph): """ runner.set_graph( graph ) sets the graph to graph """ presStack.set_graph(self, graph) center = self.env['center'] center.reset() self.nodes = {} if len(center.node_names) == 0: raise Exception("No nodes to work on") for nodename in center.node_names: proc_list = self.nodes.setdefault(nodename, []) ppn = len(proc_list) worker = Worker(nodename, self.env.current_env, processor=ppn) proc_list.append(worker) center.set_event() assert not center.done def run(self): """ required by Thread """ center = self.env['center'] try: self.main_loop() except Exception, msg: center.error = True, msg self.safe_release_lock() center.abort_all() self.join_all() raise self.join_all() center.reset()
def action(signum, stack_frame): ''' action to catch alarm ''' print print "WallTime exceded" print env = InstanceManager() env['center'].error = True
def __init__(self, inSpace, num_blocks): env = InstanceManager() np = env['slimvars']['np'] if prod(num_blocks) != np: raise Exception( "num_blocks does not match mpi variable 'np', %(num_blocks)s != %(np)s " % vars()) kparams = dict(num_blocks=tuple(num_blocks)) LinearOperatorStruct.__init__(self, inSpace, **kparams)
class Node(object): """ Node class to represent data and commands in the hash table """ env = InstanceManager() _objID = None __setstate__ = None name = "Node" def __init__(self, obj): """ take any object and store it to the hashtable while tracking it with this node instance """ table = self.env['table'] self._objID = table.append(obj) def get(self): """ get object associated with this node from the hash Table """ table = self.env['table'] return table[self._objID] def getID(self): """ return int id of node object """ return self._objID id = property(getID) def __str__(self): return "(%s:%s)" % (self.name, self._objID) def __repr__(self): return self.__str__() def __getattr__(self, attr): return getattr(self.get(), attr) def __eq__(self, other): if isinstance(other, Node) and self.id == other.id: return True else: return False
class Runner(object): """ Abstract class """ # log = Log() env = InstanceManager() cleaner = None created_nodes = None def set_graph(self, graph): raise NotImplementedError("Please use a subclass of RunnerBase") # def addSource(self, source): # """ # add a source to the graph manager # """ # self.env['graphmgr'].addSource(source) def add(self, commands): """ simple helper function TODO: replace with reduce( lambda x,y:y+y,commands) """ commands = list(commands) prev = commands.pop(0) if not commands: if prev.adder is None: return prev else: return prev + None while commands: next = commands.pop(0) prev = prev + next return prev def get_new_sources(self): if self.cleaner is None: cleaned_nodes = set() else: cleaned_nodes = self.cleaner.get_cleaned_nodes() if self.created_nodes is None: remaining = set() else: remaining = self.created_nodes.difference(cleaned_nodes) return remaining
class RecordHelper( object): env = InstanceManager() def __init__(self,verbose_level ,node, debug): self.msg_record_item = { 'verb':verbose_level, 'node':node, 'debug':debug} def write(self,msg): msg_record_item = self.msg_record_item.copy() msg_record_item['msg'] = msg self.env['record'].rec( msg_record_item )
def __new_instance__(self, name): """ simalar to init but for singleton initializes the class when a new instance is created """ Singleton.__new_instance__(self, name) # self.log = Log() self.env = InstanceManager() self.graph = DiGraph() self.__breakpoints = set() self.__sources = set() self.__targets = set() self.runner = defaultRunner() self.builder = PipeBuilder()
def build(self, target, source, env): # env['runtype'] = 'dryrun' env['verbose'] = ARGUMENTS.get('verbose', 0) debug = ARGUMENTS.get('debug', None) if debug: debug = debug.split(',') else: debug = [] env['debug'] = debug env['logfile'] = None runner = dottestRunner() from slimpy_base.Environment.InstanceManager import InstanceManager __env__ = InstanceManager() __env__['graphmgr'].setRunner(runner) self.act(target, source, env)
class BuilderBase(object): """ abstract class """ # log = Log() # table = HashTable() # slimvars = GlobalVars() env = InstanceManager() def __init__(self): pass def build(self, g, targets, sources, breakpoints, **k): raise NotImplementedError( "Builer subclass must implement build method") def isSource(self, val): """ test if val is a DataContainer or id of a DataContainer and if that DataContainer is full """ try: node = self.env['table'][val] except KeyError: return False if isinstance(node, DataContainer): return node.isfull() return False def get_graph(self): return self._graph def set_graph(self, graph): self._graph = graph graph = property(get_graph, set_graph)
class VectorFactory( object ): """ factory class to contain data into a subclass of dataContainer and put that into a vector """ __metaclass__ = Profileable """ Class used as a convinience method to contain various data formats and return a vector """ # graphmgr = GraphManager() env = InstanceManager() @note("class is initialized by user") def __init__( self ): """ nothing done in the init """ pass def __call__( self, data ): """ contain data into a subclass of dataContainer and put that into a vector """ if isinstance(data, ADI): newdata = data.container else: newdata = contain( data ) self.env['graphmgr'].add_source( newdata ) return Vector( newdata ) def __str__(self): return "<SLIMpy function: vector>" def __repr__(self): return "SLIMpy.VectorFactory( )"
def __init__(self, *arg, **kw): ''' ''' new_kw = dict(version=make_version()) new_kw.update(kw) OptionParser.__init__(self, *arg, **new_kw) self._required = set() self._types = {} self._defaults = {} self._prog_args = [] self.env = InstanceManager() self.add_all_slim_options() from slimpy_base.setup.DEFAULTS import DEFAULTS self.set_defaults(**DEFAULTS)
class WorkerTester(TestCase): env = InstanceManager() def testrun(self): worker = Worker("nodename", self.env.current_env, processor=1) # with self.env['center'] as center: # worker.start() def testsafe_release_lock(self): if NI: raise NotImplementedError("test not implemented") def test__str__(self): if NI: raise NotImplementedError("test not implemented") def testmain_loop(self): if NI: raise NotImplementedError("test not implemented") def testprint_(self): if NI: raise NotImplementedError("test not implemented") def testabort(self): if NI: raise NotImplementedError("test not implemented") def testrun_job(self): if NI: raise NotImplementedError("test not implemented") def test__init__(self): worker = Worker("nodename", self.env.current_env, processor=1) center = self.env['center'] self.failUnless(("nodename", 1) in center.idle)
class solver(object): """ Abstract class """ # slimvars = GlobalVars() env = InstanceManager() # log = env['record'] def __init__(self): pass def solve(self, A, x): """ Not ImplementedError """ raise NotImplementedError("Must subclass sovler base class") def __str__(self): return "<SLIMpy abstract solver>" def __repr__(self): return "<SLIMpy abstract solver>"
but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with SLIMpy . If not, see <http://www.gnu.org/licenses/>. """ from slimpy_base.Core.User.Structures.Scalar import * from numpy import real_if_close as numpy_real_if_close from slimpy_base.Environment.InstanceManager import InstanceManager from slimpy_base.utils.RegisterType import register_slimpy_func ## The SLIMpy global environment __env__ = InstanceManager( ) ## for * import __all__ = [ "scalar_max", "scalar_min", "real_if_close", "scalar_value", "scalar_mean" ] def generate_new_scalar( one, other, func): """ generate_new_scalar( one, other, func) -> scalar """ scalar = Scalar( ) trg_scal = Target( scalar ) src_self = Source( one )
class Vector( ADI ): """ When creating a SLIMpy Vector using the command vector(filename) several variables are initialized. The only input that the user should need is `file` corresponding to a file on disk. The other variables are mostly for internal use. If file is not None, creates a vector instance with a pointer to the file `file`. If create is given as a list on integers a vector with a temporary name and the given size will be created. There are more easy to use functions such as zeros to create a file """ ## for string name = "Vector" ## slimpy global variables env = InstanceManager() ## vector implements VectorType __metaclass__ = VectorType def __new__( cls, *args ): #IGNORE if args and isinstance(args[0], ndarray): from slimpy_base.User.AumentedMatrix.AugVector import AugVector avec = AugVector( args[0] ).ravel() for idx, container in enumerate( avec): avec[idx] = Vector( container ) return avec else: return object.__new__( cls, *args ) ## Constructor form a data container def __init__( self, container ): """ data - must be a data_container. """ ADI.__init__( self, container ) self.__transpose_flag = False self.__sorted_vector = None ########################################################################## ### OPERATOR OVERLOADS ### ########################################################################## def _get_transp_flag(self): return self.__transpose_flag def _set_transp_flag(self,val): self.__transpose_flag = bool(val) ## test if vector is transposed is_transp = property( _get_transp_flag, _set_transp_flag ) def __setitem__( self, index, item ): """ Sets an item in the vector dictionary """ raise Exception, "not implemented. Ask me to do it later" ## returns a @ref slimpy_base.Core.User.Structures.VectorSpace.VectorSpace "VectorSpace" representing this vector def getSpace( self ): """ returns a Space object that contains all of the information about this vector """ from slimpy_base.Core.User.Structures.VectorSpace import VectorSpace return VectorSpace( self.getParameters( ) ) ## getter property for getSpace space = property( getSpace ) # Overloads [i:j] Returns a __numarray array from index i to j from the binary file # def __getslice__(self,start,stop,step=1): # """ # Returns a numarray array from index i to j from the binary file. # If j is less than i or i is greater than the length of the vector, returns a # numarray array of zero elements. # Otherwise it behaves like a list. # """ # s = slice(start,stop,step) # # return self.genScalar('getslice',s) #raise Exception , "not implemented. Ask me to do it later" # Overloads [i:j] Writes a __numarray array from index i to j to the binary file ## set a slice of this data inplace def __setslice__( self, i, j, data ): """ Writes a numarray array from index i to j to the binary file corresponding to the data_container. """ raise Exception , "not implemented. Ask me to do it later" ## @name Vector Transformation Operations # # @{ ## Overload the '+' __operator if other can be a scalar or another vector # for self + other invokes math() def __add__( self, other ): """ vec1.__add__(vec2) <==> vec1 + vec2 """ if other is 0: return self if other is self: return self *2 par = other if isinstance( other, Vector ): par = Source( other.getContainer() ) return self.generateNew( 'add', vec=par ) else: par = self.source_or_num( par ) return self.generateNew( 'add', par ) ## Overload the '+' __operator for other + self def __radd__( self, other ): """ vec1.__radd__(vec2) <==> vec2 + vec1 vec2 may be a scalar or a vector """ return ( self + other ) ## Overload the '-' __operator if other can be a scalar or another vector def __neg__( self ): """ vec1.__neg__() <==> -vec1 """ return self.generateNew( 'neg' ) ## Overload the '-' __operator if other can be a scalar or another vector def __sub__( self, other ): """ self..__sub__(other) <==> self - other vec2 may be a scalar or a vector """ if other is 0: return self if other is self: return self.space.zeros() par = other if isinstance( other, Vector ): par = Source( other.getContainer() ) return self.generateNew( 'sub', vec=par ) else: par = self.source_or_num( par ) return self.generateNew( 'sub', par ) ## Overload the '-' __operator # if other can be a scalar or another vector for other - self def __rsub__( self, other ): """ vec1.__rsub__(vec2) <==> vec2 - vec1 vec2 may be a scalar or a vector """ par = other if isinstance( other, Vector ): par = Source( other.getContainer() ) return self.generateNew( 'rsub', vec=par ) else: par = self.source_or_num( par ) return self.generateNew( 'rsub', par ) ## Overload the '/' __operator if other can be a scalar or another vector def __div__( self, other ): """ vec1.__div__(vec2) <==> vec1 / vec2 vec2 may be a scalar or a vector """ if other is 1: return self if other is self: return self.space.ones() par = other if isinstance( other, Vector ): par = Source( other.getContainer() ) return self.generateNew( 'div', vec=par ) else: par = self.source_or_num( par ) return self.generateNew( 'div', par ) ## Overload the '/' __operator if other can be a scalar or another vector def __rdiv__( self, other ): """ vec1.__rdiv__(vec2) <==> vec2 / vec1 vec2 may be a scalar or a vector """ par = other if isinstance( other, Vector ): par = Source( other.getContainer() ) return self.generateNew( 'rdiv', vec=par ) else: par = self.source_or_num( par ) return self.generateNew( 'rdiv', par ) ## Overload the '*' __operator if other can be a scalar or another vector def __mul__( self, other ): """ vec1.__mul__(vec2) <==> vec1 * vec2 vec2 may be a scalar or a vector """ if other is 0: return 0 if other is 1: return self if other is self: return self ** 2 par = other if isinstance( other, Vector ): if self.is_transp ^ other.is_transp: if self.is_transp: return inner_product( self, other ) else: return outer_product( self, other ) par = Source( other.getContainer() ) return self.generateNew( 'mul', vec=par ) else: par = self.source_or_num( par ) return self.generateNew( 'mul', par ) ## Overload the '*' __operator if other can be a scalar or another vector def __rmul__( self, other ): """ vec1.__rmul__(vec2) <==> vec2 * vec """ return self.__mul__( other ) ## Overload the '**' __operator if other can be a scalar or another vector def __pow__( self, other ): """ vec1.__pow__(vec2) <==> vec1 ** vec2 vec2 may be a scalar or a vector """ if other is 1: return self par = other if isinstance( other, Vector ): par = Source( other.getContainer() ) return self.generateNew( 'pow', vec=par ) else: par = self.source_or_num( par ) return self.generateNew( 'pow', par ) ## Overload abs() def __abs__( self ): """ Vector.__abs__(vec2) <==> abs(vec1) vec2 may be a scalar or a vector """ return self.generateNew( 'abs' ) ########################################################################## # Boolean comparison operators ########################################################################## ## boolean comparison def __or__(self, other): """ vec1 | vec2 -> vec3 """ return spboolean.or_( self, other ) ## boolean comparison def __and__(self,other): "vec1 & vec2 -> vec3" return spboolean.and_( self, other ) ## boolean comparison def __lt__(self,other): """ vec1.__lt__(vec2) <==> vec1 < vec2 """ return spboolean.less_than( self, other ) ## boolean comparison def __gt__(self,other): """ vec1.__gt__(vec2) <==> vec1 > vec2 """ return spboolean.greater_than( self, other ) ## boolean comparison def __le__(self,other): """ vec1.__le__(vec2) <==> vec1 <= vec2 """ return spboolean.less_than_eq(self, other) ## boolean comparison def __ge__(self,other): """ vec1.__lt__(vec2) <==> vec1 >= vec2 """ return spboolean.greater_than_eq(self, other) ## boolean comparison def __eq__(self,other): """ vec1.__eq__(vec2) <==> vec1 == vec2 """ return spboolean.equal(self, other) ## boolean comparison def __ne__(self,other): """ vec1.__ne__(vec2) <==> vec1 != vec2 """ return spboolean.not_equal( self, other ) ## thresholding operation # @param obj can be a Scalar or vector # @param mode one of 'soft', 'hard' or 'nng' def thr( self, obj, mode='soft' ): """ Returns a thresholded vector. obj - may be a scalar or a vector. mode - may be 'soft', 'hard' or 'nng' """ par = obj if isinstance( obj, Vector ): par = Source( obj.getContainer() ) return self.generateNew( 'thr' , mode=mode, fthr=par ) else: par = self.source_or_num( par ) return self.generateNew( 'thr' , mode=mode, thr=par ) ## hard thresholding def thrhard( self, obj ): """ Returns a thresholded vector. obj - may be a scalar or a vector. Same as vector.sort(obj,'hard') """ return self.thr( obj, mode='hard' ) def garrote( self, obj ): """ Returns a garroted vector. obj - may be a scalar or a vector. Same as vector.sort(obj,'nng') """ return self.thr( obj, mode='nng' ) ## Sort a vector by absolute values # @param ascmode sort in ascending or descending order # @param memsize define memory size of system def sort( self, ascmode=False, memsize=None ): """ Returns a vector with a sorted data set. """ if memsize is None: memsize = self.env['slimvars']['memsize'] return self.generateNew( "sort", memsize=memsize, ascmode=ascmode ) ## real part of vector def __real__( self ): """ Returns a vector containing the real component of a complex vector. """ if self.space.isComplex(): return self.generateNew( 'real' ) else: return self ## imaginary part of vector def __imag__( self ): """ Returns a vector containing the imaginary component of a complex vector. """ return self.generateNew( 'imag' ) ## @} real = __real__ imag = __imag__ ########################################################################## ### END OF OPERATOR OVERLOADS ### ########################################################################## ## @name Scalar Reduction Operations # # @{ def __getitem__( self, index ): """ If index is a string: Returns an item from one of the dictionaries. Checks the vector dictionary first. If index is a number: Returns a number at index from the binary file. """ if isinstance( index, (int,Scalar) ): ind = self.source_or_num(index) return self.scalar_reduction( 'getitem', ind ) elif isinstance( index, str ): raise Exception, "not implemented. Ask me to do it later" elif isinstance( index, slice ): l = len( self.getSpace() ) ind = index.indices( l ) return self.scalar_reduction( 'getslice', ind ) else: raise TypeError , "expected int or string got %s" % type( index ) ## root mean square of vector # @return a slimpy_base.Core.User.Structures.Scalar.Scalar object def rms( self ): """ Returns the root mean square""" return self.scalar_reduction( 'rms' ) ## maximum element of vector # @return a slimpy_base.Core.User.Structures.Scalar.Scalar object def max( self ): """ Returns the maximum value""" return self.scalar_reduction( 'max' ) ## minimum element of vector # @return a slimpy_base.Core.User.Structures.Scalar.Scalar object def min( self ): """ Returns the minimum value""" return self.scalar_reduction( 'min' ) ## mean value of vector # @return a slimpy_base.Core.User.Structures.Scalar.Scalar object def mean( self ): """ Returns the mean value""" return self.scalar_reduction( 'mean' ) ## variance of vector # @return a slimpy_base.Core.User.Structures.Scalar.Scalar object def var( self ): """ Returns the variance""" return self.scalar_reduction( 'var' ) ## standard deviation of vector # @return a slimpy_base.Core.User.Structures.Scalar.Scalar object def sdt( self ): """ Returns the standard deviation""" return self.scalar_reduction( 'std' ) ## lval-norm of vector # @return a slimpy_base.Core.User.Structures.Scalar.Scalar object # @param lval power of the norm def norm( self, lval=2 ): """ Returns the lval norm of the vector """ return self.scalar_reduction( 'norm', lval ) ## ith order statistic of the vector (ith smallest element), # @param i index within size of vector def orderstat( self, i ): """ Returns the ith order statistic of the vector (ith smallest element), i.e. i=0 is smallest, i=1 second smallest, etc.) Negative indexing starts with the largest element, i.e. i=-1 is largest, i=-2 second largest, etc. """ if not 'sorted' in self.__dict__: self.sorted = self.sort( ascmode=False ) if i<0: i = len( self.getSpace() ) + i return self.sorted[i] ## @} # #=============================================================================== # DEPRECATED #=============================================================================== ## @name Deprecated # these functions have been replaced by linear operators # @{ def conj( self ): """ return vector conjugate of data """ if self.space.isComplex(): return self.generateNew( 'conj' ) else: return self ## gradient of the vector def grad( self ): """ Returns a vector containing the gradient of the vector. """ #return __comp([self.generateNew('halfderiv'),self.generateNew('halfderiv')]) return self.generateNew( 'grad' ) # NOISE def noise( self, mean=0, seed=None ): """ Returns vector with added random noise. mean - noise mean seed - noise seed """ param = dict( mean=mean, seed=seed ) if seed is None: param.pop( 'seed', None ) return self.generateNew( 'noise', **param ) ## TRANSP # @TODO: Move transp to image # @deprecated def transp( self, a1=1, a2=2, memsize=100 ): """ transpose a dataset """ if a1 == a2: return self return self.generateNew( 'transp', plane=( a1, a2 ), memsize=memsize ) ## @} def _transpose(self): vec = Vector( self.container ) vec.is_transp = not vec.is_transp return vec ## symbolic transpose of the vector # @warning does not transpose underlying image T = property( _transpose ) def _conj_transp(self): vec = Vector(self.container).conj( ) vec.is_transp = not self.is_transp return vec H = property( _conj_transp ) ## sets the name of the file on disk def setName( self, name, path=None ): """ set the name of this vector. Makes it non persistent. """ if not path is None: name = join( path, name ) container = self.getContainer() container.setName( name , path=None ) self.env['graphmgr'].add_target( container ) return self ## plot vector # @todo: should not be a vector method def plot( self, command=None, plotcmd=None ): """ calls plot method of this vectors container """ try: self.flush() except: #IGNORE:W0704 pass #IGNORE:W0702 self.container.plot( command=command, plotcmd=plotcmd ) ## reshape underlying image # @todo: should not be a vector method def reshape(self, *shape): if len(shape) is 1 and isinstance(shape[0], (list,tuple)): shape = shape[0] return self.generateNew( "reshape" , shape=shape )
SLIMpy is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with SLIMpy . If not, see <http://www.gnu.org/licenses/>. """ from unittest import TestCase, defaultTestLoader from slimpy_base.Core.Interface.Structure import Structure from slimpy_base.Core.Command.Command import Command from slimpy_base.Environment.InstanceManager import InstanceManager _env = InstanceManager() NI = True class StructureTester(TestCase): def testgenerate_command(self): struct = Structure() _env['slimvars']['keep_tb_info'] = False c1 = struct.generate_command('cmnd', a1=11) c2 = Command('cmnd', None, a1=11) self.failUnlessEqual(c1, c2)
class Scalar(object): ''' Scalar object contains class methods to work on rsf data ''' env = InstanceManager() # log = Log() re_float = re_compile('[-+]?\d*\.\d*') @classmethod def get_command(cls, cmd): '@deprecated: ' if hasattr(cls, cmd): return cls(cmd) else: raise TypeError('No Scalar command "%s" in RSF ' % cmd) def __init__(self, cmd): if not hasattr(self, cmd): raise TypeError('No Scalar command "%s" in RSF ' % cmd) self._command_name = cmd def __call__(self, container, scalar, *args, **kw): ''' only non classmethod calls methoc given by self._command_name ''' if not hasattr(self, "_command_name"): raise AttributeError("scalar class not initialized") cmd = self._command_name attr = getattr(self, cmd) container.node_copy('localhost') attr(container, scalar, *args, **kw) # slimvars = GlobalVars() def __str__(self): return "rsf scalar method %s" % self._command_name def __repr__(self): return self.__str__() @staticmethod def rms(container, scalar): """ Returns the root mean square""" num = Scalar.attr(container, scalar, want='rms') scalar.set(num) @staticmethod def max(container, scalar): """ Returns the maximum value""" num = Scalar.attr(container, scalar, want='max') scalar.set(num) @staticmethod def min(container, scalar): """ Returns the minimum value""" num = Scalar.attr(container, scalar, want='min') scalar.set(num) @staticmethod def mean(container, scalar): """ Returns the mean value""" num = Scalar.attr(container, scalar, want='mean') scalar.set(num) @staticmethod def var(container, scalar): """ Returns the variance""" num = Scalar.attr(container, scalar, want='var') scalar.set(num) @staticmethod def sdt(container, scalar): """ Returns the root""" num = Scalar.attr(container, scalar, want='std') scalar.set(num) @staticmethod def norm(container, scalar, lval=2): """ Returns the lval norm of the vector """ is_inf = lval == inf or (isinstance(lval, str) and lval.lower() == 'inf') #IGNORE:E1103 if is_inf: a1 = Scalar.attr(container, scalar, want='max') a2 = Scalar.attr(container, scalar, want='min') num = max(abs(a1), abs(a2)) else: num = Scalar.attr(container, scalar, want='norm', lval=lval) scalar.set(num) @staticmethod def attr_make_number(atr): ''' call eval on atr if result is tuple make complex @param atr: string @type atr:str ''' num = eval(atr) if isinstance(num, tuple): num = complex(*num) return num @staticmethod def attr(container, scalar, want='norm', lval=2): """ fetches various attributes of the vector. vector will be flushed prior to the operation. TODO: make it so that the vector does not flush automatically. """ # command = "%(path)s < %(data)s want=%(want)s lval=%(lval)s" %vars() command = ATTRMD(attr=SFATTR(), file=container.get_data('localhost'), want=want, lval=lval) #define parameters for the pipe print >> __env__['record'](1, 'cmd'), command p0 = Popen(command, shell=True, stdout=PIPE, stderr=PIPE) err = p0.wait() if err: lines = p0.stderr.read() p0.stderr.close() p0.stdout.close() raise IOError(err, command + '\n' + lines) output = p0.stdout.read() p0.stderr.close() p0.stdout.close() n = output.split('=')[-1].split('at')[0] return Scalar.attr_make_number(n) @staticmethod def project(container, scalar, lamb=None): ''' returns threshold value that will threshold the vector to a 1-norm of lamb ''' if lamb is None: raise TypeError('Must specify "lamb=" value') command = PROJCMD( project=SFPROJECT(), file=container.get_data('localhost'), lamb=lamb, ) p0 = Popen(command, shell=True, stdout=PIPE, stderr=PIPE) err = p0.wait() if err: lines = p0.stderr.read() p0.stderr.close() p0.stdout.close() raise IOError(err, command + '\n' + lines) n = p0.stdout.read() p0.stderr.close() p0.stdout.close() assert 'tau=' in n num = n.split('=')[-1].strip() scalar.set(float(num)) @staticmethod def getitem(container, scalar, item): "get item-th element from the vector" if not isinstance(item, int): raise TypeError, "getitem got %(item)s expected int" # params = container.getParameters( ) print >> __env__['record'](1, 'cmd'), "getitem( %s )" % (item) item = container.readbin(start=item, len=1)[0] l = lambda x: ( (x == int32 and int) or (x == float32 and float) or (x == float64 and float) or (x == complex64 and complex)) t = l(type(item)) num = t(item) scalar.set(num) return
class rsf_data_container(DataContainer): """ rsf_data_container - to keep track of "out of core" vectors corresponding binary files on disk. """ suffix = ".rsf" psuffix = ".vpl" name = "rsf" # slimvars = GlobalVars() # log = Log() env = InstanceManager() COUNT = 0 sfFactory = rsfCommandFactory() # command = "%(path)s < %(data)s want=%(want)s lval=%(lval)s" %vars() _scalar_methods = ScalarMethods @classmethod def isCompatibleWith(cls, obj): ''' statict method to determine if 'obj' is an rsf file @param obj: @type obj: any object that would be contained in a datacontainer class ''' obj = str(obj) if obj.endswith(cls.suffix): if not isfile(obj): raise Exception, "the file %s can not be found" % (obj) return True if isfile(obj + cls.suffix): return True return False def __init__(self, data=None, parameters=None, command=None, tmp=None, nodenames=None, target_node=None): # self.scalar = Scalar() self.SFRM = join(self.env['slimvars']['RSFBIN'], 'sfrm') self.SFGREY = join(self.env['slimvars']['RSFBIN'], 'sfgrey') self.SFMV = join(self.env['slimvars']['RSFBIN'], 'sfmv') self.XTPEN = join(self.env['slimvars']['RSFBIN'], 'xtpen') self.SFATTR = join(self.env['slimvars']['RSFBIN'], 'sfattr') self.SFDD = join(self.env['slimvars']['RSFBIN'], 'sfdd') self.PLOTCMD = Template("< ${data} ${plotcmd} | ${viewcmd} &") self.MVCMD = Template("${mv} ${From} ${To}") self.RMCMD = Template("${rm} ${file}") self.ATTRMD = Template("${attr} < ${file} want=${want} lval=${lval} ") node_info = None if data is not None: data = str(data) data = abspath(data) if not data.endswith(self.suffix): data = data + self.suffix dname = dirname(data) data = basename(data) node_info = dict(location=dname) DataContainer.__init__(self, data=data, parameters=parameters, command=command, tmp=tmp, nodenames=nodenames, node_info=node_info, target_node=target_node) def __setitem__(self, item, value): """ """ raise NotImplementedError # assert item == 'data', "can not set the item %s(item)s" % vars() def makeparams(self): ''' returns a param object created from the data contained ie. the rsf header info ''' par = DataContainer.makeparams(self) # RSF specific stuff try: format, data_type = par['data_format'].split('_') except KeyError: print self, 'has no key "data_format"' raise par['data_type'] = data_type return par def genname_helper(self, string): """ formatting and better output files, no more tempname just 'slim.%prog.%cmd.%id.rsf' where %prog is the main programs name and %cmd is the last command on the pipe that created the data and %id is a unique incremented identifier """ main_file = "ineractive" main = modules['__main__'] if hasattr(main, '__file__'): main_file = main.__file__ prog = basename(main_file)[:-3] join_dot = lambda *args: ".".join(args) pid = str(getpid()) cur_env = self.env.current_env make = lambda fcn, cnt: join_dot('slim', pid, cur_env, prog[:6], fcn[:5], "%.5d" % cnt) + self.suffix if exists(string): string = split(string)[1] self.__class__.COUNT += 1 filename = make(string, self.COUNT) while exists(filename): self.__class__.COUNT += 1 filename = make(string, self.COUNT) return filename def genName(self, command=None): """ generate a random name if command is given then generate a unique but formatted name 'slim.%prog.%cmd.%id.rsf' """ if command is not None and hasattr(command, 'tag') and isinstance( command.tag, str): filename = self.genname_helper(command.tag) elif command is not None and isinstance(command, str): filename = self.genname_helper(command) else: # raise Exception( "must spcify a valid command to generate an appropreate file" ) td = self.get_tmp_dir() filename = tempnam(td) + self.suffix return filename def get_tmp_dir(self): if self.is_local: return self.env['slimvars']['localtmpdir'] else: return self.env['slimvars']['globaltmpdir'] def isempty(self): """ check if this data has bee built """ local_data = self.get_data('localhost') if isfile(local_data): return False if self.data: if 'localhost' in self.nodenames: return not isfile(local_data) else: return not len(self.nodenames) # def getConverter( self , command ): # ''' # return converter class # # command must have attribute 'function' # ''' # return self.sfFactory[command.tag] @classmethod def get_converter(self, command): if isinstance(command, str): tag = command else: tag = command.tag return self.sfFactory[tag] def __str__(self): """Adds the current lib's suffix to the end of filename note: if no lib is set then self.plugin.suffix returns '' """ if self.data != None: return str(self.data) else: return "None" def __repr__(self): return str(self) def path(self): """ returns the absolute pathname to the file """ pathstr = abspath(str(self)) pathstr = dirname(pathstr) return pathstr def getName(self): 'returns the name of the data contained' return self.data def plot(self, command=None, plotcmd=None): """ plot returns the path the the plotfile """ if command is None: if plotcmd is None: plotcmd = self.SFGREY command = self.PLOTCMD.substitute(data=self.local_data, plotcmd=plotcmd, viewcmd=self.XTPEN) print >> self.env['record'](1, 'plot'), command system(command) return None def _get_local_data(self): return self.get_data("localhost") local_data = property(_get_local_data) def _get_any_node(self): if 'localhost' in self.nodenames or not self.nodenames: node = 'localhost' else: node = list(self.nodenames)[0] return node, self.get_data(node) def setName(self, newname, path=None): """wrapped by SLIMpy.serial_vector.setname""" newname = str(newname) if newname.endswith(self.suffix): newname = newname else: newname = newname + self.suffix if not path: newname = abspath(newname) path = dirname(newname) newname = basename(newname) if self.isfull(): self.move(newname) self.updateData(newname) ninfo = self._node_info.setdefault('localhost', {}) self.add_node_name('localhost') ninfo['location'] = path self.tmp(False) def updateData(self, newname): ''' @param newname: string to rename data to @type newname: ''' self.data = newname def getHeader(self): """ return open header, data must exist """ node, data = self._get_any_node() if node == 'localhost': return open(data) else: cat = Unix_Pipes.CreateComand(['cat'], node_name=node, source=data) p0 = Popen(cat, shell=True, stdout=__PIPE__) return p0.stdout def move(self, newname): "move data file on disk" mv = self.MVCMD.substitute(mv=self.SFMV, From=self.local_data, To=abspath(newname)) err = system(mv) if err is not 0: raise Exception("commmand %(mv)s failed " % vars()) def rm(self): """ removes the file on disc """ print >> self.env['record']( 10, 'cleaner'), "call to remove %s:" % self.data print >> self.env['record']( 11, 'cleaner'), "\tistmp=%s , isfull=%s" % (self.istmp(), self.isfull()) if not self.istmp(): return False if self.isempty(): return False err = 0 cmd_log1 = self.env['record'](1, 'cmd') # print "rm called" synccmd = self.env['slimvars']['SYNC_CMD'] do_sync = self.env['slimvars']['sync_disk_om_rm'] def sync(synccmd): sync_command = Unix_Pipes.CreateComand([synccmd], node) print >> cmd_log1, sync_command p0 = Popen(sync_command, shell=True, stderr=__PIPE__) ret = p0.wait() if ret: print >> self.env[ 'record'], 'error running %(sync_command)s ' % vars() print >> self.env[ 'record'], 'try running SLIMpy with "sync_disk_om_rm=False" ' % vars( ) for node in self.nodenames.copy(): data = self.get_data(node) sfrm = self.RMCMD.substitute(rm=self.SFRM, file=data) rm = self.RMCMD.substitute(rm='rm', file=data) if do_sync: sync(synccmd) command = Unix_Pipes.CreateComand([sfrm], node) print >> cmd_log1, command p0 = Popen(command, shell=True, stderr=__PIPE__) ret = p0.wait() print >> self.env['record'](2), "finished::", node, 'rm' if ret: err += 1 msg = p0.stderr.read() p0.stderr.close() print >> self.env[ 'record'], 'error %(ret)s on %(command)s: removeing header file:\n%(msg)s' % vars( ) command = Unix_Pipes.CreateComand([rm], node) print >> cmd_log1, command p0 = Popen(command, shell=True, stderr=__PIPE__) ret = p0.wait() if ret: msg = p0.stderr.read() print >> cmd_log1, "could not 'sfrm' or rm 'data'\n%(msg)s" % vars( ) else: self.nodenames.remove(node) else: self.nodenames.remove(node) p0.stderr.close() return not err def node_copy(self, node_name): """ copy from one node to another """ if self.is_global or self.isempty(): return if node_name in self.nodenames: return tmddir = self.env['slimvars']['localtmpdir'] from_cmd = "%s form=xdr" % self.SFDD to_cmd = "%s form=native" % self.SFDD from_node = list(self.nodenames)[0] from_data = self.get_data(from_node) to_data = self.get_data(node_name) is_tmp = self.istmp() to_cmd = Unix_Pipes.CreateComand([to_cmd], node_name=node_name, target=to_data, is_local=[True], is_tmp=[is_tmp], ssh_localhost=True) frm_node = list(self.nodenames)[0] self.add_node_name(node_name) cmd = Unix_Pipes.CreateComand([from_cmd, to_cmd], is_local=[None, None], is_tmp=[is_tmp, is_tmp], node_name=frm_node, source=from_data, ssh_localhost=False) print >> self.env['record'](1, 'cmd'), cmd assert frm_node != node_name, "can not copy from same node to same node" p0 = Popen(cmd, shell=True, stderr=__PIPE__) p0.wait() print >> self.env['record'](2, 'cmd'), "finished:: cp", node_name if p0.returncode: out = p0.stderr.read() raise IOError("copying from node to node retuned error %s\n\n%s" % (p0.returncode, out)) def available_from(self, nodename): if nodename in self.nodenames: return True elif 'localhost' in self.nodenames: return True return False def readattr(self): 'returns rsf header info as a dict object' header = self.getHeader() lines = header.readlines() header.close() file_dict = {} for line in lines: if '=' in line: line = line[1:-1] line = line.split('=') try: line[1] = eval(line[1]) except: pass file_dict[line[0]] = line[1] self.BinFile = file_dict.pop('in') #l = lambda x : (x=='native_int' and int) or (x=='native_float' and float) or (x=='native_double' and long) or (x=='native_complex' and complex) #file_dict['type'] = l(file_dict['data_format']) #del file_dict['data_format'] return file_dict def readbin(self, start=0, len=-1): #IGNORE:W0622 ''' @precondition: data must exist. @return: a numpy array. @param start: element in data to start from. @type start: int @param: len: lenght of data to read into array if -1 reads until the end of data. @type len:int ''' l = lambda x: ((x == 'native_int' and int32) or (x == 'native_float' and float32) or (x == 'native_double' and float64) or (x == 'native_complex' and complex64)) file_dict = self.readattr() typeflag = l(file_dict['data_format']) fd = open(self.BinFile) fd.seek(start * typeflag().nbytes) return fromfile(fd, typeflag, len) def writebin(self, data, start=0, size=0): """ write data to file not tested """ header = self.path() start = start * 4 vector_dict, file_dict = self.readattr(header) if size == 0: size = product(vector_dict['dim']) binfile = file_dict['in'] fd = open(binfile, 'r+') #a = array.array('f', __numarray.transpose(data[:size]).tolist()[0]) fd.seek(start) data[:, :size].tofile(fd) fd.close() def book(self, header): 'not tested' header = str(header) f = open(header) L = f.read() return L def writeattr(self, header, vector_dict, file_dict): 'not tested' file_dict = dict(file_dict) if file_dict.has_key('in'): del file_dict['in'] if file_dict.has_key('data_format'): del file_dict['data_format'] for i in range(1, len(vector_dict['dim']) + 1): file_dict['n%s' % i] = vector_dict['dim'][i - 1] fd = self.readattr(header) for item in fd.items(): try: if file_dict[item[0]] == item[1]: del file_dict[item[0]] except: pass book = '' for item in file_dict.items(): book += "\t%s=%s\n" % (item[0], item[1]) fd = open(header, 'r+') fd.seek(0, 2) fd.write(book) fd.close() def write(self, header, data, book): 'not tested' header = str(header) fd = open(header, 'w') fd.write(book) fd.close() dict = self.readattr(header) fd = open(dict['in'], 'w') fd.close() self.writebin(header, data) def get_data(self, nodename): data = self.data if self.is_global: loc = self.get_data_loc('localhost') else: loc = self.get_data_loc(nodename) return join(loc, data) def get_data_loc(self, nodename): if nodename == 'localhost': path = self.env['slimvars']['globaltmpdir'] else: path = self.env['slimvars']['localtmpdir'] node_info = self._node_info.setdefault(nodename, {}) path = node_info.setdefault('location', path) return path def diagnostic(self, nodename=None): """ run a check if this data is valid """ log = self.env['record'](1, 'cmd', 'err') log2 = self.env['record'](2, 'diagnostic') print >> log print >> log, "Runnind diagnostic on data %s" % self print >> log acmd = self.ATTRMD.substitute(attr=self.SFATTR, file=str(self), want='all', lval=2) if nodename is None: nodename = list(self.nodenames) elif isinstance(nodename, str): nodename = [nodename] did_run = False for node in nodename: did_run = True attr_command = Unix_Pipes.CreateComand([acmd], node) print >> log2, attr_command p0 = Popen(attr_command, shell=True, stderr=_STDOUT, stdout=__PIPE__) ret = p0.wait() lines = p0.stdout.read() print >> log2, lines if ret: print >> log, 'Error running Diagnostic on data "%(self)s"' % vars( ) return False else: print >> log, 'File "%(self)s" is OK.' % vars() return did_run
class sfConverter(Converter): """ Base rsf converter class used when specific command can not be found @postcondition: tansform returns voidspace """ env = InstanceManager() # slimvars = GlobalVars() @classmethod def place_adder(cls, command): command.adder = rsfAddCommands() return command @classmethod def map(cls, source, command): cmnd = cls.default_function(command) return cls.pack(cmnd) @classmethod def guess_exe_path(cls, name, error=0): if name.startswith('sf'): sfname = name else: sfname = "sf" + name slimvars = cls.env['slimvars'] sfname = join(slimvars['RSFBIN'], sfname) if is_executable(name): return abspath(name) elif is_executable(sfname): return sfname elif WhereIs(name): return WhereIs(name) elif error: raise EnvironmentError("No files '%(name1)s' or '%(name2)s'" % vars()) else: return name @classmethod def default_function(cls, command, name=None): slimvars = cls.env['slimvars'] if name is None: name = command.tag else: name = str(name) exe = cls.guess_exe_path(name, error=slimvars['check_path']) command.func = OutOfCoreDriver(exe) command.adder = rsfAddCommands() return command @classmethod def mpi_function(cls, command, name=None, num_proc='all'): slimvars = cls.env['slimvars'] MPICOM = Template(slimvars['MPIFLAGS']) if name is None: name1 = command.tag else: name1 = name name2 = join(slimvars['RSFBIN'], 'xsf' + name1) if isfile(name1): if isabs(name1): command_name = name1 else: command_name = join(".", name1) else: command_name = name2 if slimvars['check_path'] and not isfile(command_name): raise EnvironmentError("No files '%(name1)s' or '%(name2)s'" % vars()) # assert slimvars['mpi'], "called mpi map while slimpy mpi var is false" if "$NODEFILE" in slimvars['MPIFLAGS'] and slimvars['NODEFILE'] is None: raise Exception( "SLIMpy detected an mpi process to be run needs a nodefile: " "please set 'NODEFILE' global variable") mpi_cmd = MPICOM.substitute(**slimvars.slimGlobals) cmd = "%(mpi_cmd)s %(command_name)s" % vars() command.func = OutOfCoreDriver(cmd) command.adder = rsfAddCommands() command.command_type = 'multi_process' command.num_proc = num_proc return command @classmethod def constr(cls, command, *Spaces): return True
class JobPosting(object): ''' posting class for worker threads to get info about new jobs Like a message board. ''' env = InstanceManager() def __init__(self, name, processor): self.lock = RLock() self.event = Event() self.lock.acquire() self.name = name self.processor = processor self.finished_jobs = set() self.current = set() self.todo = set() self._is_waiting = False self._first_wait = True self.start_time = 0 self.stop_time = 0 self.current_rec = None self.finished_rec = set() self.lock.release() def _get_waiting(self): 'true if worker is waiting for a job' return self._is_waiting def _set_waiting(self, val): self._is_waiting = val waiting = property(_get_waiting, _set_waiting) def get_time_since_start(self): 'get the time from the start of the job' if self.current_rec is None: return 0 else: return time.time() - self.current_rec.created time_passed = property(get_time_since_start) def notify(self): 'notify the listening worker' self.event.set() def new_todo(self, job): 'add a job to the message board' self.lock.acquire() self.todo.add(job) self.lock.release() def get(self): 'get a job from the todo pile' self.lock.acquire() job = self.todo.pop() self.current.add(job) self.current_rec = JobRecord(job, self.name, self.processor) self.current_rec.start() self.env['record'].add_job(self.current_rec) self.lock.release() return job def finished(self, job): ''' adds current job to the finished pile ''' self.lock.acquire() self.current.remove(job) self.finished_jobs.add(str(job)) rec = self.current_rec rec.stop() self.finished_rec.add(rec) print >> self.env['record']( 10, 'time'), "Command execution time: %0.4f seconds" % ( rec.finished - rec.created) self.current_rec = None self.lock.release() def is_working(self): 'true if the worker is working on a job' return len(self.current) def has_todo(self): 'returns the len of the todo pile' return len(self.todo) def busy(self): 'true if job is working or has stuff todo' self.lock.acquire() ret = self.is_working() or self.has_todo() self.lock.release() return ret def post(self, job): 'post a job to the todo' self.lock.acquire() self.todo.add(job) self.event.set() self.lock.release() def start_timer(self): 'starts timer ' self.start_time = time.time() def stop_timer(self): 'stops timer ' self.stop_time = time.time() def wait_for_job(self): "wait for slef.notify to be called" self.event.wait() self.event.clear() return def acquire(self): self.lock.acquire() def release(self): self.lock.release() def total_time(self): "total time between start and stop" return self.stop_time - self.start_time def time_idol(self): 'total time spent idle' tot = self.total_time() for jobrecord in self.finished_rec: tot -= jobrecord.total_time() return tot
class RemoteDriver(object): """ used to chain ooc_driver commands together """ env = InstanceManager() _stderr_logfile = "stderr_logfile.log" def _get_stderr_logfile(self): return join(self.env['slimvars']['localtmpdir'], self._stderr_logfile) stderr_logfile = property(_get_stderr_logfile) def __init__(self): self.__cmnd = [] self.__Target = None self.__Source = None self._node_name = None return def get_num_proc(self): numbers = [com.get_nub_proc() for com in self.getCmnd()] return max(numbers) def getCmnd(self): return self.__cmnd def get_command_list(self): return self.__cmnd def addCommand(self, command): self.setTarget(None) return self.__cmnd.append(command) def is_muti_proc(self): for cmd in self.get_command_list(): if cmd.command_type == 'multi_process': return True return False def setSource(self, src): assert not self.getCmnd( ), "command already has attributes to it, cannot add a source" self.__Source = src return def getSource(self): return self.__Source def setTarget(self, tgt): self.__Target = tgt return def getTarget(self): return self.__Target def get_node_name(self): return self._node_name def set_node_name(self, val): self._node_name = val for cmd in self.get_command_list(): cmd.node_name = val source = property(getSource, setSource) target = property(getTarget, setTarget) node_name = property(get_node_name, set_node_name) def get_targets(self): tgts = set() if self.target: tgts.add(self.target) for com in self.__cmnd: tgts.update(com._get_target_cont()) return tgts def make_locals_list(self): lst = [] push = lst.append for com in self.get_command_list(): push(None) for target in com._get_target_cont(): if target.is_global: lst[-1] = False break else: lst[-1] = True if lst[-1] is False: pass # do nothing target is already global elif self.target: # set it to the location of the stdout target is_local = bool(self.node_name != 'localhost') lst[-1] = is_local return lst def make_tmp_list(self): lst = [] push = lst.append for com in self.__cmnd: push(None) for target in com._get_target_cont(): if target.istmp(): lst[-1] = True break else: lst[-1] = False if self.target: lst[-1] = self.target.istmp() return lst def get_sources(self): srcs = set() if self.source: srcs.add(self.source) for com in self.__cmnd: srcs.update(com._get_source_cont()) return srcs def __str__(self): format = [ com.func.format(self.node_name, com.params, com.kparams) for com in self.getCmnd() ] command = " | ".join(format) if self.source: command = "< %s %s" % (self.source, command) if self.target: command = "%s > %s" % (command, self.target) return command # return "< %s %s > %s" %( self.getSource(), command , self.getTarget() ) def __call__(self, *params, **kparams): """ runs driver with pipe. uses the list of """ return self.run() def run(self): if self.node_name is None: raise Exception("job host not set") log = self.env['record'](5, 'cmd', 'err') # print >> log # print >> log, "command failed, running diagnostic" # print >> log try: self._run() except IOError: print >> log, "command failed, running diagnostic" if self.env['slimvars']['run_again']: if self.diagnose(): print >> log, "All diagnostics passed: running command again" self._run() else: print >> log, "diagnostic failed: raising exception" raise else: raise return def diagnose(self): for source in self.get_sources(): if not source.diagnostic(): return False return True def add_node_name_to_targets(self): if self.target: if self.target.is_local: self.target.add_node_name(self.node_name) else: self.target.add_node_name('localhost') for cmd in self.get_command_list(): cmd.add_node_name_to_targets() return def copy_sources(self): """ recursively copy source nodes if necessary """ if self.source and self.source.is_local: if self.is_muti_proc(): self.source.node_copy('localhost') else: self.source.node_copy(self.node_name) for cmd in self.get_command_list(): cmd.copy_sources() def _run(self): center = self.env['center'] self.copy_sources() center.acquire() cmd = self.pipe_format() log = self.env['record'] print >> log(5, 'cmd'), cmd p3 = Popen(cmd, stderr=__PIPE__, shell=True) pid = p3.pid center.add_pid(pid) # p3.tochild.close() center.release() try: err = p3.wait() print >> log(5), "finished::", self.node_name except OSError, e: if e.errno is 10: err = p3.returncode else: raise center.acquire() self.add_node_name_to_targets() center.remove_pid(pid) lines = p3.stderr.readlines() last_line = "".join(lines) #@UnusedVariable if err: # last_line = self.get_last_line_of_err_log() node = self.node_name #@UnusedVariable center.release() raise IOError( err, "process %(pid)s on node %(node)s, returned %(err)s: \n%(last_line)s\nCommand:%(cmd)s" % vars()) else: center.release() if last_line: print >> log(5, 'cmd', 'err'), last_line p3.stderr.close() return
class PSpace(object): """ parameter is a simple tracking method to pass Metadata without being bound to each specific datatype methods are similar to a python dict """ ## @var _space_dict # @brief dict of information about this space ## @var _plugin # @brief slimpy_base.api.Plugin represented by this space ## slimpy global variables env = InstanceManager() ## Constructor, create a space from another space or like a dict object # @param plugin a string name or a subclass of DataContainer def __init__(self, plugin, *E, **D): """ create a space from another space """ if isinstance(plugin, PSpace): self._space_dict = dict(plugin.params, *E, **D) self.plugin = plugin.plugin else: self._space_dict = dict(*E, **D) self.plugin = plugin ## Get the plugin represented by this space def _get_plugin(self): 'Get the plug-in represented by this space' return self._plugin def _set_plugin(self, plugin): from slimpy_base.Core.Interface.ContainerBase import DataContainer plugin_type = type(plugin) assert plugin_type == str or issubclass(plugin, DataContainer) if isinstance(plugin, str): self._plugin = self.env['keystone'].getplugin(plugin) else: self._plugin = plugin ## get and set property for self._plugin plugin = property(_get_plugin, _set_plugin) ## Make a new container from the current parameters # @param command may be a string or a slimpy_base.Core.Command.Command.Command # @tmp if the contaier is a temporary file def makeContaner(self, command=None, tmp=None): """ makeContaner( self, command=None ,tmp=None) -> Container Make a new container from the current parameters """ return self.plugin(parameters=self, command=command, tmp=tmp) ## create a new PSpace updated with new keys def newParameter(self, *E, **F): """ Make a new parameter updated with the new keys, 'keys' """ SpaceType = type(self) x = SpaceType(self.plugin, self, *E, **F) return x ## update this PSpace with another def update(self, *E, **F): """ update this PSpace with another same as a python dict """ for e in E: if isinstance(e, PSpace): self._space_dict.update(e.params) elif e is None: pass else: self._space_dict.update(e) self._space_dict.update(**F) return ## number of elements the space represents def __len__(self): return self.size ## shallow copy def copy(self): """ returns a shallow copy of self """ SpaceType = type(self) return SpaceType(self.plugin, self._space_dict.copy()) ## returns tr # @return bool def has_key(self, k): ' D.has_key(k) -> True if D has a key k, else False' return self._space_dict.has_key(k) ## shape of the underlying image def _get_shape(self): """ Returns a list of the dimensions of the image of the underlying vector """ shp = [] N = 1 while self.has_key("n%s" % N): val = self["n%s" % N] if val < 1: raise TypeError( "shape parameter does not conform to SLIMpy standard:\n" "Should be an int greater than 0") shp.append(val) N += 1 if N is 1: return () i = len(shp) - 1 while i > 0 and shp[i] is 1: shp.pop(-1) i -= 1 return shp ## setter for shape of the underlying image # @param shape is a sequence of integers def _set_shape(self, shape): """ delete all n* keys and replace them with shape """ N = 1 nN = "n%s" % N while self.has_key(nN): del self[nN] N += 1 nN = "n%s" % N for N, val in enumerate(shape): nN = "n%s" % (N + 1) self[nN] = val ## returns size of field of scalars def get_size(self): mul = lambda x, y: x * y prod = lambda shape: reduce(mul, shape, 1) shape = self.shape for i in shape: if i == UnknownValue: return None if not shape: return 0 else: return prod(shape) ## size property size = property(get_size) ## shape property shape = property(_get_shape, _set_shape) def _get_ndim(self): return len(self.shape) ndim = property(_get_ndim) ## list of keys def keys(self): return self._space_dict.keys() ## pop a key from the dict # @exception raises a KeyError if k is not a key in self and no default is given def pop(self, k, *default): return self._space_dict.pop(k, *default) def _get_params(self, keep_unknown=False): if keep_unknown: params = self._space_dict.copy() else: itemset = [(k, v) for (k, v) in self.iteritems() if v is not UnknownValue] params = dict(itemset) return params params = property(_get_params) ## @see dict def iteritems(self): """ iterates over key value pairs in self """ return self._space_dict.iteritems() ## helper for intersection and union def _space_helper(self, other, error=False): set_self = set(self.keys()) set_other = set(other.keys()) intersect = set_self.intersection(set_other) diff_self = set_self.difference(set_other) diff_other = set_other.difference(set_self) new = {} for key in intersect: v1 = self[key] v2 = other[key] if not self.equal_keys(other, key): if error: raise ValueError( "item self[%(key)s] != other[%(key)s] ; %(v1)s != %(v2)s" ) else: v1 = UnknownValue v2 = UnknownValue if v1 == UnknownValue: new[key] = v2 else: new[key] = v1 SpaceType = type(self) inter_space = SpaceType(self.plugin, new) dspace_self = SpaceType(self.plugin, [(key, self[key]) for key in diff_self]) dspace_other = SpaceType(self.plugin, [(key, other[key]) for key in diff_other]) return inter_space, dspace_self, dspace_other ## intersection of this space and another def intersection(self, other): """ space.intersection( other ) -> Space returns a Space object that is the Intersection of self and other contains the restrictions of each space the shared restrictions in each must be equal: i.e. if self contains restriction n1=5 and other -> n1=6 intersection method will raise error. """ inter_space, dspace_self, dspace_other = self._space_helper( other, True) inter_space.update(dspace_self, dspace_other) return inter_space def _union(self, other): """ space.union( other ) -> Space returns the union of this space with another """ inter_space, dspace_self, dspace_other = self._space_helper( other, False) return inter_space ## union of this space with another def union(self, other, *others): """ space.union( other, *others ) -> Space returns the union of this space with another that is only the restrictions that are the same in both spaces """ next = self._union(other) for oths in others: next = next._union(oths) return next def __eq__(self, other): if type(self) != type(other): return False test1 = self._space_dict == other._space_dict test2 = self.plugin == other.plugin return test1 and test2 ## True if self and other both have key k # @param other PSpace object # @param accept_unknown if the value at key k for either self or other is an slimpy_base.Core.Interface.PSpace.UnknownValue # then equal_keys returns accept_unknown def equal_keys(self, other, k, accept_unknown=True): """ space.equal_keys( space2, key, accept_unknown=True ) -> bool returns true if key is present and equal in both spaces. If one value is UnknownValue then will return accept_unknown. """ if self.has_key(k) and other.has_key(k): v1 = self[k] v2 = other[k] if v1 is UnknownValue or v2 is UnknownValue: return accept_unknown else: return v1 == v2 else: return False def itervalues(self): return self._space_dict.itervalues() def values(self): return self._space_dict.values() def iterkeys(self): return self._space_dict.iterkeys() def setdefault(self, k, D=None): return self._space_dict.setdefault(k, D=D) def has_unknown(self): return len([v for v in self.itervalues() if v == UnknownValue]) ## true if self is a subspace of other. def is_subspace(self, other, accept_unknown=True): """ space.is_subspace(other) <--> space in other returns true if self is a subspace of other. """ assert isinstance(other, PSpace) if self.plugin != other.plugin: return False key_set = set(self.keys()) key_set_other = set(other.keys()) issubset = key_set_other.issubset(key_set) if not issubset: return False for key in key_set_other: if not self.equal_keys(other, key, accept_unknown=accept_unknown): return False return True ## true if self is a superspace of other. def is_superspace(self, other, accept_unknown=True): return not self.is_subspace(other, accept_unknown=accept_unknown) ## true if self is a contains k # @param k a PSpace a key in self or a vector def __contains__(self, k): """ S.__contains__( k ) <-> k in S """ if isinstance(k, PSpace): return k.is_subspace(self) elif hasattr(k, 'space') and isinstance(k.space, PSpace): return k.space.is_subspace(self) else: return self._space_dict.__contains__(k) def __getitem__(self, item): return self._space_dict[item] def __delitem__(self, item): self._space_dict.__delitem__(item) def get(self, k, default=None): return self._space_dict.get(k, default) def __setitem__(self, item, val): self._space_dict[item] = val def __repr__(self): name = self.__class__.__name__ return "%s( %s, %s )" % (name, repr(self.plugin), repr( self._space_dict)) def __str__(self): dtype = self.get('data_type', "unknown data type") size = self.size name = self.__class__.__name__ return "<%(name)s %(dtype)s:%(size)s>" % vars()