def _checkpointGen(filePath,orCall,force,unpack,useNpy,*args,**kwargs):
# XXX assume pickling now, ends with 'npz'
# if the file from 'filePath' exists and 'force' is false, loads the file
# otherwise, calls 'orCall' and saves the result. *args and **kwargs
# are passed to 'orCall'.
# 'Unpack' unpacks the array upon a load. This makes it 'look' like a
# simple function call (returns the args, or a tuple list of args)
# use unpack if you aren't dealing with dictionaries or things like that
if pGenUtil.isfile(filePath) and not force:
if (useNpy):
return _npyLoad(filePath,unpack)
else:
# assume we pickle in binary
fh = open(filePath,'rb')
data = pickle.load(fh)
fh.close()
return data
else:
# couldn't find the file.
# make sure it exists
path = pGenUtil.getBasePath(filePath)
pGenUtil.ensureDirExists(path)
# POST: we can put our file here
dataToSave = orCall(*args,**kwargs)
# need to figure out if we need to unpack all the arguments..
if (useNpy):
_npySave(filePath,dataToSave)
else:
# open the file in binary format for writing
with open(filePath, 'wb') as fh:
pickle.dump(dataToSave,fh)
return dataToSave
def _checkpointGen(filePath, orCall, force, unpack, useNpy, *args, **kwargs):
# XXX assume pickling now, ends with 'npz'
# if the file from 'filePath' exists and 'force' is false, loads the file
# otherwise, calls 'orCall' and saves the result. *args and **kwargs
# are passed to 'orCall'.
# 'Unpack' unpacks the array upon a load. This makes it 'look' like a
# simple function call (returns the args, or a tuple list of args)
# use unpack if you aren't dealing with dictionaries or things like that
if pGenUtil.isfile(filePath) and not force:
if (useNpy):
return _npyLoad(filePath, unpack)
else:
# assume we pickle in binary
fh = open(filePath, 'rb')
data = pickle.load(fh)
fh.close()
return data
else:
# couldn't find the file.
# make sure it exists
path = pGenUtil.getBasePath(filePath)
pGenUtil.ensureDirExists(path)
# POST: we can put our file here
dataToSave = orCall(*args, **kwargs)
# need to figure out if we need to unpack all the arguments..
if (useNpy):
_npySave(filePath, dataToSave)
else:
# open the file in binary format for writing
with open(filePath, 'wb') as fh:
pickle.dump(dataToSave, fh)
return dataToSave
def saveFile(filePath, dataToSave, useNpy):
path = pGenUtil.getBasePath(filePath)
pGenUtil.ensureDirExists(path)
# need to figure out if we need to unpack all the arguments..
if (useNpy):
_npySave(filePath, dataToSave)
else:
# open the file in binary format for writing
with open(filePath, 'wb') as fh:
# XXX make protocol specifiable?
pickle.dump(dataToSave, fh, pickle.HIGHEST_PROTOCOL)
def saveFile(filePath,dataToSave,useNpy):
path = pGenUtil.getBasePath(filePath)
pGenUtil.ensureDirExists(path)
# need to figure out if we need to unpack all the arguments..
if (useNpy):
_npySave(filePath,dataToSave)
else:
# open the file in binary format for writing
with open(filePath, 'wb') as fh:
# XXX make protocol specifiable?
pickle.dump(dataToSave,fh,pickle.HIGHEST_PROTOCOL)
