def _checkpointGen(filePath,orCall,force,unpack,useNpy,*args,**kwargs): # XXX assume pickling now, ends with 'npz' # if the file from 'filePath' exists and 'force' is false, loads the file # otherwise, calls 'orCall' and saves the result. *args and **kwargs # are passed to 'orCall'. # 'Unpack' unpacks the array upon a load. This makes it 'look' like a # simple function call (returns the args, or a tuple list of args) # use unpack if you aren't dealing with dictionaries or things like that if pGenUtil.isfile(filePath) and not force: if (useNpy): return _npyLoad(filePath,unpack) else: # assume we pickle in binary fh = open(filePath,'rb') data = pickle.load(fh) fh.close() return data else: # couldn't find the file. # make sure it exists path = pGenUtil.getBasePath(filePath) pGenUtil.ensureDirExists(path) # POST: we can put our file here dataToSave = orCall(*args,**kwargs) # need to figure out if we need to unpack all the arguments.. if (useNpy): _npySave(filePath,dataToSave) else: # open the file in binary format for writing with open(filePath, 'wb') as fh: pickle.dump(dataToSave,fh) return dataToSave
def _checkpointGen(filePath, orCall, force, unpack, useNpy, *args, **kwargs): # XXX assume pickling now, ends with 'npz' # if the file from 'filePath' exists and 'force' is false, loads the file # otherwise, calls 'orCall' and saves the result. *args and **kwargs # are passed to 'orCall'. # 'Unpack' unpacks the array upon a load. This makes it 'look' like a # simple function call (returns the args, or a tuple list of args) # use unpack if you aren't dealing with dictionaries or things like that if pGenUtil.isfile(filePath) and not force: if (useNpy): return _npyLoad(filePath, unpack) else: # assume we pickle in binary fh = open(filePath, 'rb') data = pickle.load(fh) fh.close() return data else: # couldn't find the file. # make sure it exists path = pGenUtil.getBasePath(filePath) pGenUtil.ensureDirExists(path) # POST: we can put our file here dataToSave = orCall(*args, **kwargs) # need to figure out if we need to unpack all the arguments.. if (useNpy): _npySave(filePath, dataToSave) else: # open the file in binary format for writing with open(filePath, 'wb') as fh: pickle.dump(dataToSave, fh) return dataToSave
def filter(self): force = self._force if (pGenUtil.isfile(self._filePath) and not force): # return a new checkpoint object from the data data = np.load(self._filePath) return CheckpointData(**data) else: idx = self.getFilterIdx() return DataFilter.filterDataStatic(self._data,idx, self._filePath,force)
def _checkpointGen(filePath, orCall, force, unpack, useNpy, *args, **kwargs): # XXX assume pickling now, ends with 'npz' # if the file from 'filePath' exists and 'force' is false, loads the file # otherwise, calls 'orCall' and saves the result. *args and **kwargs # are passed to 'orCall'. # 'Unpack' unpacks the array upon a load. This makes it 'look' like a # simple function call (returns the args, or a tuple list of args) # use unpack if you aren't dealing with dictionaries or things like that if pGenUtil.isfile(filePath) and not force: return loadFile(filePath, useNpy) else: # couldn't find the file. # make sure it exists # POST: we can put our file here dataToSave = orCall(*args, **kwargs) # save the data, so next time we can just load saveFile(filePath, dataToSave, useNpy) return dataToSave
def _checkpointGen(filePath,orCall,force,unpack,useNpy,*args,**kwargs): # XXX assume pickling now, ends with 'npz' # if the file from 'filePath' exists and 'force' is false, loads the file # otherwise, calls 'orCall' and saves the result. *args and **kwargs # are passed to 'orCall'. # 'Unpack' unpacks the array upon a load. This makes it 'look' like a # simple function call (returns the args, or a tuple list of args) # use unpack if you aren't dealing with dictionaries or things like that if pGenUtil.isfile(filePath) and not force: return loadFile(filePath,useNpy) else: # couldn't find the file. # make sure it exists # POST: we can put our file here dataToSave = orCall(*args,**kwargs) # save the data, so next time we can just load saveFile(filePath,dataToSave,useNpy) return dataToSave
def pipeline(objects, force=None): # objects are a list, each element is : [<file>,<function>,<args>]: # file name, # function then the ('extra' args the funcion # needs. we assume that each filter in the pipeline takes # the previous arguments, plus any others, and returns the next arg # the first just takes in whatever it is given, the last can return anything # in other words, the signatures are: # f1(f1_args), returning f2_chain # f2(f2_chain,f2_args), returning f3_chain # ... # fN(fN_chain,fNargs), returning whatever. filesExist = [pGenUtil.isfile(o[pipe_fileIdx]) for o in objects] numObjects = len(objects) # get a list of forces force = _pipeListParser(force, False, numObjects) # get a list of how to save. numpy = [not o[pipe_fileIdx].endswith('.pkl') for o in objects] # by default, if no force arguments passed, assume we dont want to force # in other words: just load by default runIfFalse = [ fExists and (not forceThis) for fExists, forceThis in zip(filesExist, force) ] if (False not in runIfFalse): # just load the last... otherArgs = _pipeHelper(objects[-1], False, numpy[-1]) else: # need to run at least one, go through them all otherArgs = None firstZero = runIfFalse.index(False) # if not at the start, load 'most downstream' if (firstZero != 0): idx = firstZero - 1 otherArgs = _pipeHelper(objects[idx], force[idx], numpy[idx], otherArgs) # POST: otherargs is set up, if we need it. for i in range(firstZero, numObjects): otherArgs = _pipeHelper(objects[i], force[i], numpy[i], otherArgs) return otherArgs
def pipeline(objects,force=None): # objects are a list, each element is : [<file>,<function>,<args>]: # file name, # function then the ('extra' args the funcion # needs. we assume that each filter in the pipeline takes # the previous arguments, plus any others, and returns the next arg # the first just takes in whatever it is given, the last can return anything # in other words, the signatures are: # f1(f1_args), returning f2_chain # f2(f2_chain,f2_args), returning f3_chain # ... # fN(fN_chain,fNargs), returning whatever. filesExist = [pGenUtil.isfile(o[pipe_fileIdx]) for o in objects] numObjects = len(objects) # get a list of forces force = _pipeListParser(force,False,numObjects) # get a list of how to save. numpy = [ not o[pipe_fileIdx].endswith('.pkl') for o in objects] # by default, if no force arguments passed, assume we dont want to force # in other words: just load by default runIfFalse = [ fExists and (not forceThis) for fExists,forceThis in zip(filesExist,force)] if (False not in runIfFalse): # just load the last... otherArgs = _pipeHelper(objects[-1],False,numpy[-1]) else: # need to run at least one, go through them all otherArgs = None firstZero = runIfFalse.index(False) # if not at the start, load 'most downstream' if (firstZero != 0): idx = firstZero-1 otherArgs = _pipeHelper(objects[idx], force[idx],numpy[idx], otherArgs) # POST: otherargs is set up, if we need it. for i in range(firstZero,numObjects): otherArgs = _pipeHelper(objects[i],force[i],numpy[i],otherArgs) return otherArgs
def callIfNoFile(cls,toCall,fileN): if (pGenUtil.isfile(fileN) ): return np.load(fileN) else: return toCall(fileN)