def _checkpointGen(filePath,orCall,force,unpack,useNpy,*args,**kwargs):
# XXX assume pickling now, ends with 'npz'
# if the file from 'filePath' exists and 'force' is false, loads the file
# otherwise, calls 'orCall' and saves the result. *args and **kwargs
# are passed to 'orCall'.
# 'Unpack' unpacks the array upon a load. This makes it 'look' like a
# simple function call (returns the args, or a tuple list of args)
# use unpack if you aren't dealing with dictionaries or things like that
if pGenUtil.isfile(filePath) and not force:
if (useNpy):
return _npyLoad(filePath,unpack)
else:
# assume we pickle in binary
fh = open(filePath,'rb')
data = pickle.load(fh)
fh.close()
return data
else:
# couldn't find the file.
# make sure it exists
path = pGenUtil.getBasePath(filePath)
pGenUtil.ensureDirExists(path)
# POST: we can put our file here
dataToSave = orCall(*args,**kwargs)
# need to figure out if we need to unpack all the arguments..
if (useNpy):
_npySave(filePath,dataToSave)
else:
# open the file in binary format for writing
with open(filePath, 'wb') as fh:
pickle.dump(dataToSave,fh)
return dataToSave
def _checkpointGen(filePath, orCall, force, unpack, useNpy, *args, **kwargs):
# XXX assume pickling now, ends with 'npz'
# if the file from 'filePath' exists and 'force' is false, loads the file
# otherwise, calls 'orCall' and saves the result. *args and **kwargs
# are passed to 'orCall'.
# 'Unpack' unpacks the array upon a load. This makes it 'look' like a
# simple function call (returns the args, or a tuple list of args)
# use unpack if you aren't dealing with dictionaries or things like that
if pGenUtil.isfile(filePath) and not force:
if (useNpy):
return _npyLoad(filePath, unpack)
else:
# assume we pickle in binary
fh = open(filePath, 'rb')
data = pickle.load(fh)
fh.close()
return data
else:
# couldn't find the file.
# make sure it exists
path = pGenUtil.getBasePath(filePath)
pGenUtil.ensureDirExists(path)
# POST: we can put our file here
dataToSave = orCall(*args, **kwargs)
# need to figure out if we need to unpack all the arguments..
if (useNpy):
_npySave(filePath, dataToSave)
else:
# open the file in binary format for writing
with open(filePath, 'wb') as fh:
pickle.dump(dataToSave, fh)
return dataToSave
def filter(self):
force = self._force
if (pGenUtil.isfile(self._filePath) and not force):
# return a new checkpoint object from the data
data = np.load(self._filePath)
return CheckpointData(**data)
else:
idx = self.getFilterIdx()
return DataFilter.filterDataStatic(self._data,idx,
self._filePath,force)
def _checkpointGen(filePath, orCall, force, unpack, useNpy, *args, **kwargs):
# XXX assume pickling now, ends with 'npz'
# if the file from 'filePath' exists and 'force' is false, loads the file
# otherwise, calls 'orCall' and saves the result. *args and **kwargs
# are passed to 'orCall'.
# 'Unpack' unpacks the array upon a load. This makes it 'look' like a
# simple function call (returns the args, or a tuple list of args)
# use unpack if you aren't dealing with dictionaries or things like that
if pGenUtil.isfile(filePath) and not force:
return loadFile(filePath, useNpy)
else:
# couldn't find the file.
# make sure it exists
# POST: we can put our file here
dataToSave = orCall(*args, **kwargs)
# save the data, so next time we can just load
saveFile(filePath, dataToSave, useNpy)
return dataToSave
def _checkpointGen(filePath,orCall,force,unpack,useNpy,*args,**kwargs):
# XXX assume pickling now, ends with 'npz'
# if the file from 'filePath' exists and 'force' is false, loads the file
# otherwise, calls 'orCall' and saves the result. *args and **kwargs
# are passed to 'orCall'.
# 'Unpack' unpacks the array upon a load. This makes it 'look' like a
# simple function call (returns the args, or a tuple list of args)
# use unpack if you aren't dealing with dictionaries or things like that
if pGenUtil.isfile(filePath) and not force:
return loadFile(filePath,useNpy)
else:
# couldn't find the file.
# make sure it exists
# POST: we can put our file here
dataToSave = orCall(*args,**kwargs)
# save the data, so next time we can just load
saveFile(filePath,dataToSave,useNpy)
return dataToSave
def pipeline(objects, force=None):
# objects are a list, each element is : [<file>,<function>,<args>]:
# file name,
# function then the ('extra' args the funcion
# needs. we assume that each filter in the pipeline takes
# the previous arguments, plus any others, and returns the next arg
# the first just takes in whatever it is given, the last can return anything
# in other words, the signatures are:
# f1(f1_args), returning f2_chain
# f2(f2_chain,f2_args), returning f3_chain
# ...
# fN(fN_chain,fNargs), returning whatever.
filesExist = [pGenUtil.isfile(o[pipe_fileIdx]) for o in objects]
numObjects = len(objects)
# get a list of forces
force = _pipeListParser(force, False, numObjects)
# get a list of how to save.
numpy = [not o[pipe_fileIdx].endswith('.pkl') for o in objects]
# by default, if no force arguments passed, assume we dont want to force
# in other words: just load by default
runIfFalse = [
fExists and (not forceThis)
for fExists, forceThis in zip(filesExist, force)
]
if (False not in runIfFalse):
# just load the last...
otherArgs = _pipeHelper(objects[-1], False, numpy[-1])
else:
# need to run at least one, go through them all
otherArgs = None
firstZero = runIfFalse.index(False)
# if not at the start, load 'most downstream'
if (firstZero != 0):
idx = firstZero - 1
otherArgs = _pipeHelper(objects[idx], force[idx], numpy[idx],
otherArgs)
# POST: otherargs is set up, if we need it.
for i in range(firstZero, numObjects):
otherArgs = _pipeHelper(objects[i], force[i], numpy[i], otherArgs)
return otherArgs
def pipeline(objects,force=None):
# objects are a list, each element is : [<file>,<function>,<args>]:
# file name,
# function then the ('extra' args the funcion
# needs. we assume that each filter in the pipeline takes
# the previous arguments, plus any others, and returns the next arg
# the first just takes in whatever it is given, the last can return anything
# in other words, the signatures are:
# f1(f1_args), returning f2_chain
# f2(f2_chain,f2_args), returning f3_chain
# ...
# fN(fN_chain,fNargs), returning whatever.
filesExist = [pGenUtil.isfile(o[pipe_fileIdx]) for o in objects]
numObjects = len(objects)
# get a list of forces
force = _pipeListParser(force,False,numObjects)
# get a list of how to save.
numpy = [ not o[pipe_fileIdx].endswith('.pkl') for o in objects]
# by default, if no force arguments passed, assume we dont want to force
# in other words: just load by default
runIfFalse = [ fExists and (not forceThis)
for fExists,forceThis in zip(filesExist,force)]
if (False not in runIfFalse):
# just load the last...
otherArgs = _pipeHelper(objects[-1],False,numpy[-1])
else:
# need to run at least one, go through them all
otherArgs = None
firstZero = runIfFalse.index(False)
# if not at the start, load 'most downstream'
if (firstZero != 0):
idx = firstZero-1
otherArgs = _pipeHelper(objects[idx],
force[idx],numpy[idx],
otherArgs)
# POST: otherargs is set up, if we need it.
for i in range(firstZero,numObjects):
otherArgs = _pipeHelper(objects[i],force[i],numpy[i],otherArgs)
return otherArgs
def callIfNoFile(cls,toCall,fileN):
if (pGenUtil.isfile(fileN) ):
return np.load(fileN)
else:
return toCall(fileN)
