def test_basic_mpi_functionality(self):
"""
Test basic MPI functionality
"""
temp_input_filename = os.path.join(SIRISAACDIR,
"temporary_input_{}.dat".format(os.getpid()))
temp_output_filename = os.path.join(SIRISAACDIR,
"temporary_output_{}.dat".format(os.getpid()))
temp_stdout_filename = os.path.join(SIRISAACDIR,
"temporary_stdout_{}.txt".format(os.getpid()))
input_data = {'test':123,'output_filename':temp_output_filename}
save(input_data,temp_input_filename)
stdoutFile = open(temp_stdout_filename,'w')
subprocess.call([ "mpirun","-np",str(NUMPROCS),"python",
os.path.join(SIRISAACDIR, "mpi_basic.py"),
temp_input_filename ],
stderr=stdoutFile,stdout=stdoutFile,
env=os.environ)
output_data = load(temp_output_filename)
os.remove(temp_input_filename)
os.remove(temp_output_filename)
os.remove(temp_stdout_filename)
self.assertTrue(input_data['test'] == output_data['test_output'])
def subsetsWithFits(fileNumString,onlyNew=False):
"""
Find data subsets (N) that have models that have been fit to
all conditions.
onlyNew (False) : Optionally include only subsets that have
fits that are not included in the current
combined fitProbs.
"""
fpd = loadFitProbData(fileNumString)
saveFilename = fpd.values()[0]['saveFilename']
Nlist = []
for N in scipy.sort(fpd.keys()):
# find models that have been fit to all conditions
if len(fpd[N]['fitProbDataList']) == 1:
fitModels = fpd[N]['fitProbDataList'][0]['logLikelihoodDict'].keys()
else:
fitModels = scipy.intersect1d([ fp['logLikelihoodDict'].keys() \
for fp in fpd[N]['fittingProblemList'] ])
if onlyNew:
Nfilename = directoryPrefixNonly(fileNumString,N)+'/'+saveFilename
fileExists = os.path.exists(Nfilename)
if not fileExists: # no combined file exists
if len(fitModels) > 0:
Nlist.append(N)
else: # check which fit models are currently included in the saved file
fpMultiple = load(Nfilename)
fitModelsSaved = fpMultiple.logLikelihoodDict.keys()
if len(scipy.intersect1d(fitModels,fitModelsSaved)) < len(fitModels):
Nlist.append(N)
else:
if len(fitModels) > 0:
Nlist.append(N)
return Nlist
def loadFitProbData(fileNumString):
try:
fitProbData = load(fileNumString + '_fitProbData.dat')
except (IOError, EOFError):
print "loadFitProbData: WARNING Unable to load fitProbData file."\
"Returning None."
fitProbData = None
return fitProbData
def loadFitProbData(fileNumString):
try:
fitProbData = load(fileNumString+'_fitProbData.dat')
except (IOError, EOFError):
print "loadFitProbData: WARNING Unable to load fitProbData file."\
"Returning None."
fitProbData = None
return fitProbData
def subsetsWithFits(fileNumString, onlyNew=False):
"""
Find data subsets (N) that have models that have been fit to
all conditions.
onlyNew (False) : Optionally include only subsets that have
fits that are not included in the current
combined fitProbs.
"""
fpd = loadFitProbData(fileNumString)
saveFilename = fpd.values()[0]['saveFilename']
Nlist = []
for N in scipy.sort(fpd.keys()):
# find models that have been fit to all conditions
if len(fpd[N]['fitProbDataList']) == 1:
fitModels = fpd[N]['fitProbDataList'][0]['logLikelihoodDict'].keys(
)
else:
fitModels = scipy.intersect1d([ fp['logLikelihoodDict'].keys() \
for fp in fpd[N]['fittingProblemList'] ])
if onlyNew:
Nfilename = directoryPrefixNonly(fileNumString,
N) + '/' + saveFilename
fileExists = os.path.exists(Nfilename)
if not fileExists: # no combined file exists
if len(fitModels) > 0:
Nlist.append(N)
else: # check which fit models are currently included in the saved file
fpMultiple = load(Nfilename)
fitModelsSaved = fpMultiple.logLikelihoodDict.keys()
if len(scipy.intersect1d(fitModels,
fitModelsSaved)) < len(fitModels):
Nlist.append(N)
else:
if len(fitModels) > 0:
Nlist.append(N)
return Nlist
def loadFitProb(saveFilename, fileNumString, conditioni, numTimepoints):
dirPrefix = directoryPrefix(fileNumString, conditioni, numTimepoints)
return load(dirPrefix + saveFilename)[numTimepoints]
def combineFitProbs(fileNumString,
saveCombined=True,
combinedLean=True,
reset=False):
"""
Combine fittingProblems from multiple conditions saved in the
parallel file structure into a single fittingProblemDict.
Currently only includes data from models that have been fit to
all conditions.
saveCombined (True) : Overwrites any current top-level fitProbDict file
with a combined fitProbDict containing all
numTimepoints. Set to False to minimize memory use.
combinedLean (True) : Combined fpd is saved without models to save memory.
reset (False) : If True, overwrite or delete any existing combined
fitProbDicts. This erases any existing
outOfSampleCost information.
"""
fitProbData = loadFitProbData(fileNumString)
saveFilename = fitProbData.values()[0]['saveFilename']
#save({},saveFilename)
if saveCombined: fpdMultiple = {}
fitSubsets = subsetsWithFits(fileNumString)
subsetsToCombine = subsetsWithFits(fileNumString, onlyNew=not reset)
for numTimepoints in fitSubsets:
Nfilename = directoryPrefixNonly(fileNumString,
numTimepoints) + '/' + saveFilename
fileExists = os.path.exists(Nfilename)
if fileExists and reset:
# then an old combined file exists -- erase it to reset
os.remove(Nfilename)
fileExists = False
print "combineFitProbs: Reset removed file for numTimepoints =", numTimepoints
if numTimepoints in subsetsToCombine: # combine
oldOutOfSampleCostDict = {}
if fileExists: # grab any out-of-sample cost data
fpMultiple = load(Nfilename)
if hasattr(fpMultiple, 'outOfSampleCostDict'):
oldOutOfSampleCostDict = fpMultiple.outOfSampleCostDict
p = fitProbData[numTimepoints]
fpList = []
for conditioni in range(len(p['fitProbDataList'])):
fp = loadFitProb(saveFilename, fileNumString, conditioni,
numTimepoints)
fpList.append(fp)
# make new multiple condition fitting problem by starting
# with an empty fitting problem and inserting the fittingProblemList
saveKey = p['saveKey']
fp.stopFittingN = p['stopFittingN']
fpMultiple = FittingProblemMultipleCondition([], [],
saveFilename=None,
saveKey=saveKey,
fp0=fp)
fpMultiple.fittingProblemList = fpList
fpMultiple.outOfSampleCostDict = oldOutOfSampleCostDict
# Populate the logLikelihoodDict, etc by running fitAll.
fpMultiple.fitAll(onlyCombine=True)
if saveCombined: fpdMultiple[numTimepoints] = fpMultiple
save(fpMultiple, Nfilename)
print "combineFitProbs: Done with numTimepoints =", numTimepoints
else: # no new fits to combine; just load from file
if saveCombined: fpdMultiple[numTimepoints] = load(Nfilename)
print "combineFitProbs: Done with numTimepoints =", numTimepoints
if saveCombined:
if combinedLean:
makeFpdLean(fpdMultiple)
save(fpdMultiple, saveFilename[:-4] + '_combined.dat')
def loadFitProb(saveFilename,fileNumString,conditioni,numTimepoints):
dirPrefix = directoryPrefix(fileNumString,conditioni,numTimepoints)
return load(dirPrefix+saveFilename)[numTimepoints]
def combineFitProbs(fileNumString,saveCombined=True,combinedLean=True,
reset=False):
"""
Combine fittingProblems from multiple conditions saved in the
parallel file structure into a single fittingProblemDict.
Currently only includes data from models that have been fit to
all conditions.
saveCombined (True) : Overwrites any current top-level fitProbDict file
with a combined fitProbDict containing all
numTimepoints. Set to False to minimize memory use.
combinedLean (True) : Combined fpd is saved without models to save memory.
reset (False) : If True, overwrite or delete any existing combined
fitProbDicts. This erases any existing
outOfSampleCost information.
"""
fitProbData = loadFitProbData(fileNumString)
saveFilename = fitProbData.values()[0]['saveFilename']
#save({},saveFilename)
if saveCombined: fpdMultiple = {}
fitSubsets = subsetsWithFits(fileNumString)
subsetsToCombine = subsetsWithFits(fileNumString,onlyNew=not reset)
for numTimepoints in fitSubsets:
Nfilename = directoryPrefixNonly(fileNumString,numTimepoints)+'/'+saveFilename
fileExists = os.path.exists(Nfilename)
if fileExists and reset:
# then an old combined file exists -- erase it to reset
os.remove(Nfilename)
fileExists = False
print "combineFitProbs: Reset removed file for numTimepoints =",numTimepoints
if numTimepoints in subsetsToCombine: # combine
oldOutOfSampleCostDict = {}
if fileExists: # grab any out-of-sample cost data
fpMultiple = load(Nfilename)
if hasattr(fpMultiple,'outOfSampleCostDict'):
oldOutOfSampleCostDict = fpMultiple.outOfSampleCostDict
p = fitProbData[numTimepoints]
fpList = []
for conditioni in range(len(p['fitProbDataList'])):
fp = loadFitProb(saveFilename,fileNumString,conditioni,numTimepoints)
fpList.append(fp)
# make new multiple condition fitting problem by starting
# with an empty fitting problem and inserting the fittingProblemList
saveKey = p['saveKey']
fp.stopFittingN = p['stopFittingN']
fpMultiple = FittingProblemMultipleCondition([],[],saveFilename=None,
saveKey=saveKey,fp0=fp)
fpMultiple.fittingProblemList = fpList
fpMultiple.outOfSampleCostDict = oldOutOfSampleCostDict
# Populate the logLikelihoodDict, etc by running fitAll.
fpMultiple.fitAll(onlyCombine=True)
if saveCombined: fpdMultiple[numTimepoints] = fpMultiple
save(fpMultiple,Nfilename)
print "combineFitProbs: Done with numTimepoints =",numTimepoints
else: # no new fits to combine; just load from file
if saveCombined: fpdMultiple[numTimepoints] = load(Nfilename)
print "combineFitProbs: Done with numTimepoints =",numTimepoints
if saveCombined:
if combinedLean:
makeFpdLean(fpdMultiple)
save(fpdMultiple,saveFilename[:-4]+'_combined.dat')