times,yeastRawData,params = simulateYeastOscillator([0,time],T, \
initialConditions=initialConditions)
speciesData.append(yeastRawData[:7, -1])
derivsData.append(yeastRawData[-7:, -1])
else: # 7.25.2012 keep all timepoints
times,yeastRawData,params = simulateYeastOscillator([0,maxTime],T, \
initialConditions=initialConditions)
speciesData.extend(scipy.transpose(yeastRawData[:7]))
derivsData.extend(scipy.transpose(yeastRawData[-7:]))
if True:
speciesData = (scipy.transpose(speciesData)[:numSpecies]).T
derivsData = (scipy.transpose(derivsData)[:numSpecies]).T
else:
speciesData = speciesData[:numSpecies]
derivsData = derivsData[:numSpecies]
save((speciesData, derivsData), dataFilename)
# 7.19.2012 try using all data from single IC
#speciesData = yeastRawData[:numSpecies]
#derivsData = yeastRawData[-7:(-7+numSpecies)]
offset = 10 # 8.22.2012 staying away from zero is better?
indepParamsList = ICsList + offset
speciesData = speciesData + 10
# (one data point for each IC)
sigmas = 0.1 * scipy.ones_like(speciesData)
fittingData = [ dict( zip( visibleSpeciesNames, \
[ dict( [(randTime,(data,sigma))] ) \
for data,sigma in zip(dataRow,sigmaRow)] ) ) \
for randTime,dataRow,sigmaRow in zip(randTimes,speciesData,sigmas) ]
def yeastData(numPoints,timeInterval, \
numICs,useDerivs=True,includedIndices=range(7), \
timesSeed=3,noiseSeed=4,ICseed=5, \
multiplicativeErrorBar=0.1,upperRangeMultiple=1., \
randomX=True,inputVars=None,nonzeroMin=True, \
yeastSavedDataFile=None,includeNoise=True):
"""
upperRangeMultiple (1.) : Each range of initial conditions is
expanded by this factor by increasing
the upper limit.
nonzeroMin (True) : Use the lower limit for initial
conditions given in SchValJen11;
otherwise use 0. as lower limit.
"""
usePreloadedData = True #<<<<<<<*******************
allNames = scipy.array(['S1', 'S2', 'S3', 'S4', 'N2', 'A3', 'S4ex'])
names = allNames[includedIndices]
varList = names
defaultTemperature = 286.5 #changed 10.16.2013 (was 288 K before)
# Table 2 of RuoChrWol03
defaultICs = scipy.array([1.187, 0.193, 0.050, 0.115, 0.077, 2.475,
0.077]) # mM
# choose input variable(s)
if False: # use temp. as input
inputVars = ['Temperature']
inputList = [[278], [288], [293]]
inputDescriptors = [inputs[0] for inputs in inputList]
elif False: # only a single temperature value
inputVars = ['Temperature']
inputList = [[defaultTemperature]]
inputDescriptors = [inputs[0] for inputs in inputList]
elif inputVars is None: # use varying initial conditions on all 7 species
inputVars = [name + "_init" for name in names]
print "inputVars =", inputVars
# taken from SchValJen11 Table 2
ICranges = scipy.array( \
[[0.15,1.60],[0.19,2.16], \
[0.04,0.20],[0.10,0.35], \
[0.08,0.30],[0.14,2.67],[0.05,0.10]] )[includedIndices] # mM
# as I vary the number of ICs, I want the same sequence of random ICs
scipy.random.seed(ICseed)
randICmults = []
for i in range(numICs):
randICmults.append(scipy.rand(len(includedIndices)))
#randomICs = scipy.rand(numICs,len(includedIndices))* \
# (ICranges[:,1]-ICranges[:,0]) + ICranges[:,0]
randomICs = scipy.array(randICmults)*upperRangeMultiple* \
(ICranges[:,1]-nonzeroMin*ICranges[:,0]) + nonzeroMin*ICranges[:,0]
inputList = randomICs
inputDescriptors = range(len(inputList))
else:
raise Exception, "Changing inputVars not yet implemented"
# as I vary the number of ICs, I want the same sequence of lists of times
scipy.random.seed(timesSeed)
timesRandList = []
for i in range(numICs):
timesRandList.append(scipy.rand(numPoints))
# all other random stuff is done now except for the noise
# as I vary the number of ICs, I want the same sequence of noise
scipy.random.seed(noiseSeed)
# set up preloaded data dict
if yeastSavedDataFile is None:
yeastSavedDataFile = "yeastExampleData.fittingData"
yeastDict = {}
if usePreloadedData:
try:
#yeastTimes,loadedICs,yeastRawData = \
# simplePickle.load(yeastDataFilename)
yeastDict = simplePickle.load(yeastSavedDataFile)
except IOError:
print "yeastData: Could not load " + yeastSavedDataFile
usePreloadedData = False
yeastOscillatorData = {}
yeastOscillatorDataDerivs = {}
for inputs, inputDescriptor, timesRand in zip(inputList, inputDescriptors,
timesRandList):
stopAfterRunningMATLAB = False
# deal with independent parameters
# if it's temperature (only)
if inputVars is ['Temperature']:
temperature = inputs[0]
else:
temperature = defaultTemperature
initialConditions = copy.deepcopy(defaultICs)
for inputVar, input in zip(inputVars, inputs):
# if it's initial conditions
if inputVar[-5:] == "_init":
index = pylab.find(allNames == inputVar[:-5])[0]
initialConditions[index] = input
print "initialConditions =", initialConditions
eps = (timeInterval[1] - timeInterval[0]
) / 1000. #1e-3 #1e-4 # (minutes) resolution of timepoints
integratorTimes = scipy.arange(timeInterval[0], timeInterval[1] + eps,
eps)
if not randomX:
desiredTimes = scipy.linspace(timeInterval[0], timeInterval[1],
numPoints)
else:
desiredTimes = timesRand * (timeInterval[1]-timeInterval[0]) \
+ timeInterval[0]
yeastDataKey = copy.deepcopy( \
(tuple(initialConditions),inputDescriptor,tuple(desiredTimes)))
runMATLAB = not usePreloadedData
if usePreloadedData:
if yeastDict.has_key(yeastDataKey):
yeastTimes, loadedICs, yeastRawData = yeastDict[yeastDataKey]
if not scipy.all(loadedICs == initialConditions):
print "loadedICs =", loadedICs
print "initialConditions =", initialConditions
raise Exception, "loadedICs != initialConditions"
else:
print "yeastData: "+yeastSavedDataFile+" does not " \
"contain the necessary data."
runMATLAB = True
if runMATLAB:
print "yeastData: Running simulateYeastOscillator."
yeastTimes,yeastRawData,yeastParams = \
simulateYeastOscillator(integratorTimes,temperature, \
initialConditions=initialConditions)
# the integrator returns more timepoints than I want
desiredIndices = [ pylab.find(abs(yeastTimes-time)<eps/2)[0] \
for time in desiredTimes ]
yeastRawData = yeastRawData[:, desiredIndices]
yeastTimes = yeastTimes[:, desiredIndices]
yeastDict[yeastDataKey] = (yeastTimes, initialConditions,
yeastRawData)
if False: # OLD! use temp. as input, calculate v1 as output, mult. error bars
S1data = yeastRawData[0]
A3data = yeastRawData[5]
k1, K1, q = yeastParams[1], yeastParams[10], yeastParams[9]
v1data = k1 * S1data * A3data / (1. + (A3data / K1)**q)
yeastOscillatorData[inputDescriptor] = {}
yeastOscillatorData[inputDescriptor]['v1'] = \
dict( zip(times, zip(v1data,multiplicativeErrorBar*v1data)) )
varList = ['v1']
if useDerivs: # use derivatives as output (7-dimensional), const. error bars
# 2.22.2012 from Table 2 of SchValJen11
# 11.29.2012 using these even though they are stddevs of values, not derivs
stddevs = [0.4872, 0.6263, 0.0503, 0.0814, 0.0379, 0.7478,
0.0159] # mM
stddevs = scipy.array(stddevs)[includedIndices]
#derivNames = [ 'ddt_'+name for name in names ]
derivDataList = yeastRawData[7:][includedIndices]
yeastOscillatorDataDerivs[inputDescriptor] = {}
for derivName, derivData, stddev in zip(names, derivDataList,
stddevs):
constErrorBar = multiplicativeErrorBar * stddev
yeastOscillatorDataDerivs[inputDescriptor][derivName] = \
dict( zip(yeastTimes, \
zip(derivData,scipy.ones_like(derivData)*constErrorBar)) )
#varList = names
# 10.3.2011
if True: # use all species as output (7-dimensional), const. error bars
# 2.22.2012 from Table 2 of SchValJen11
stddevs = [0.4872, 0.6263, 0.0503, 0.0814, 0.0379, 0.7478,
0.0159] # mM
#includeNoise = False #True # <<<<****** 12.5.2012 to avoid negative values
verbose = True
dataList = scipy.array(yeastRawData)[:7][includedIndices]
stddevs = scipy.array(stddevs)[includedIndices]
# 11.17.2011 reorder stuff
#names,dataList = names[::-1],dataList[::-1]
yeastOscillatorData[inputDescriptor] = {}
for name, data, stddev in zip(names, dataList, stddevs):
#constErrorBar = multiplicativeErrorBar*scipy.mean(data) #*data[0]
constErrorBar = multiplicativeErrorBar * stddev
if includeNoise:
noise = scipy.random.normal(scale=constErrorBar,
size=len(data))
data = data + noise
yeastOscillatorData[inputDescriptor][name] = \
dict( zip(yeastTimes, \
zip(data,scipy.ones_like(data)*constErrorBar )) )
#if usePreloadedData:
# save data for future use via usePreloadedData
simplePickle.save(yeastDict, yeastSavedDataFile)
#Plotting.plot(yeastTimes,v1data,'o-',label=str(temperature))
if (not usePreloadedData) and stopAfterRunningMATLAB: die
fittingData = [yeastOscillatorData[d] for d in inputDescriptors]
if useDerivs:
fittingDataDerivs = [
yeastOscillatorDataDerivs[d] for d in inputDescriptors
]
else:
fittingDataDerivs = None
return fittingData, fittingDataDerivs, inputVars, inputList
def yeastData(numPoints,timeInterval, \
numICs,useDerivs=True,includedIndices=range(7), \
timesSeed=3,noiseSeed=4,ICseed=5, \
multiplicativeErrorBar=0.1,upperRangeMultiple=1., \
randomX=True,inputVars=None,nonzeroMin=True, \
yeastSavedDataFile=None,includeNoise=True):
"""
upperRangeMultiple (1.) : Each range of initial conditions is
expanded by this factor by increasing
the upper limit.
nonzeroMin (True) : Use the lower limit for initial
conditions given in SchValJen11;
otherwise use 0. as lower limit.
"""
usePreloadedData = True #<<<<<<<*******************
allNames = scipy.array(['S1','S2','S3','S4','N2','A3','S4ex'])
names = allNames[includedIndices]
varList = names
defaultTemperature = 286.5 #changed 10.16.2013 (was 288 K before)
# Table 2 of RuoChrWol03
defaultICs = scipy.array([1.187,0.193,0.050,0.115,0.077,2.475,0.077]) # mM
# choose input variable(s)
if False: # use temp. as input
inputVars = ['Temperature']
inputList = [[278],[288],[293]]
inputDescriptors = [inputs[0] for inputs in inputList]
elif False: # only a single temperature value
inputVars = ['Temperature']
inputList = [[defaultTemperature]]
inputDescriptors = [inputs[0] for inputs in inputList]
elif inputVars is None: # use varying initial conditions on all 7 species
inputVars = [name+"_init" for name in names]
print "inputVars =",inputVars
# taken from SchValJen11 Table 2
ICranges = scipy.array( \
[[0.15,1.60],[0.19,2.16], \
[0.04,0.20],[0.10,0.35], \
[0.08,0.30],[0.14,2.67],[0.05,0.10]] )[includedIndices] # mM
# as I vary the number of ICs, I want the same sequence of random ICs
scipy.random.seed(ICseed)
randICmults = []
for i in range(numICs):
randICmults.append(scipy.rand(len(includedIndices)))
#randomICs = scipy.rand(numICs,len(includedIndices))* \
# (ICranges[:,1]-ICranges[:,0]) + ICranges[:,0]
randomICs = scipy.array(randICmults)*upperRangeMultiple* \
(ICranges[:,1]-nonzeroMin*ICranges[:,0]) + nonzeroMin*ICranges[:,0]
inputList = randomICs
inputDescriptors = range(len(inputList))
else:
raise Exception, "Changing inputVars not yet implemented"
# as I vary the number of ICs, I want the same sequence of lists of times
scipy.random.seed(timesSeed)
timesRandList = []
for i in range(numICs):
timesRandList.append(scipy.rand(numPoints))
# all other random stuff is done now except for the noise
# as I vary the number of ICs, I want the same sequence of noise
scipy.random.seed(noiseSeed)
# set up preloaded data dict
if yeastSavedDataFile is None:
yeastSavedDataFile = "yeastExampleData.fittingData"
yeastDict = {}
if usePreloadedData:
try:
#yeastTimes,loadedICs,yeastRawData = \
# simplePickle.load(yeastDataFilename)
yeastDict = simplePickle.load(yeastSavedDataFile)
except IOError:
print "yeastData: Could not load "+yeastSavedDataFile
usePreloadedData = False
yeastOscillatorData = {}
yeastOscillatorDataDerivs = {}
for inputs,inputDescriptor,timesRand in zip(inputList,inputDescriptors,timesRandList):
stopAfterRunningMATLAB = False
# deal with independent parameters
# if it's temperature (only)
if inputVars is ['Temperature']:
temperature = inputs[0]
else:
temperature = defaultTemperature
initialConditions = copy.deepcopy(defaultICs)
for inputVar,input in zip(inputVars,inputs):
# if it's initial conditions
if inputVar[-5:] == "_init":
index = pylab.find(allNames==inputVar[:-5])[0]
initialConditions[index] = input
print "initialConditions =",initialConditions
eps = (timeInterval[1]-timeInterval[0])/1000. #1e-3 #1e-4 # (minutes) resolution of timepoints
integratorTimes = scipy.arange(timeInterval[0],timeInterval[1]+eps,eps)
if not randomX:
desiredTimes = scipy.linspace(timeInterval[0],timeInterval[1],numPoints)
else:
desiredTimes = timesRand * (timeInterval[1]-timeInterval[0]) \
+ timeInterval[0]
yeastDataKey = copy.deepcopy( \
(tuple(initialConditions),inputDescriptor,tuple(desiredTimes)))
runMATLAB = not usePreloadedData
if usePreloadedData:
if yeastDict.has_key(yeastDataKey):
yeastTimes,loadedICs,yeastRawData = yeastDict[yeastDataKey]
if not scipy.all(loadedICs == initialConditions):
print "loadedICs =",loadedICs
print "initialConditions =",initialConditions
raise Exception, "loadedICs != initialConditions"
else:
print "yeastData: "+yeastSavedDataFile+" does not " \
"contain the necessary data."
runMATLAB = True
if runMATLAB:
print "yeastData: Running simulateYeastOscillator."
yeastTimes,yeastRawData,yeastParams = \
simulateYeastOscillator(integratorTimes,temperature, \
initialConditions=initialConditions)
# the integrator returns more timepoints than I want
desiredIndices = [ pylab.find(abs(yeastTimes-time)<eps/2)[0] \
for time in desiredTimes ]
yeastRawData = yeastRawData[:,desiredIndices]
yeastTimes = yeastTimes[:,desiredIndices]
yeastDict[yeastDataKey] = (yeastTimes,initialConditions,yeastRawData)
if False: # OLD! use temp. as input, calculate v1 as output, mult. error bars
S1data = yeastRawData[0]
A3data = yeastRawData[5]
k1,K1,q = yeastParams[1],yeastParams[10],yeastParams[9]
v1data = k1*S1data*A3data/(1.+(A3data/K1)**q)
yeastOscillatorData[inputDescriptor] = {}
yeastOscillatorData[inputDescriptor]['v1'] = \
dict( zip(times, zip(v1data,multiplicativeErrorBar*v1data)) )
varList = ['v1']
if useDerivs: # use derivatives as output (7-dimensional), const. error bars
# 2.22.2012 from Table 2 of SchValJen11
# 11.29.2012 using these even though they are stddevs of values, not derivs
stddevs = [0.4872,0.6263,0.0503,0.0814,0.0379,0.7478,0.0159] # mM
stddevs = scipy.array(stddevs)[includedIndices]
#derivNames = [ 'ddt_'+name for name in names ]
derivDataList = yeastRawData[7:][includedIndices]
yeastOscillatorDataDerivs[inputDescriptor] = {}
for derivName,derivData,stddev in zip(names,derivDataList,stddevs):
constErrorBar = multiplicativeErrorBar*stddev
yeastOscillatorDataDerivs[inputDescriptor][derivName] = \
dict( zip(yeastTimes, \
zip(derivData,scipy.ones_like(derivData)*constErrorBar)) )
#varList = names
# 10.3.2011
if True: # use all species as output (7-dimensional), const. error bars
# 2.22.2012 from Table 2 of SchValJen11
stddevs = [0.4872,0.6263,0.0503,0.0814,0.0379,0.7478,0.0159] # mM
#includeNoise = False #True # <<<<****** 12.5.2012 to avoid negative values
verbose = True
dataList = scipy.array(yeastRawData)[:7][includedIndices]
stddevs = scipy.array(stddevs)[includedIndices]
# 11.17.2011 reorder stuff
#names,dataList = names[::-1],dataList[::-1]
yeastOscillatorData[inputDescriptor] = {}
for name,data,stddev in zip(names,dataList,stddevs):
#constErrorBar = multiplicativeErrorBar*scipy.mean(data) #*data[0]
constErrorBar = multiplicativeErrorBar*stddev
if includeNoise:
noise = scipy.random.normal(scale=constErrorBar,size=len(data))
data = data + noise
yeastOscillatorData[inputDescriptor][name] = \
dict( zip(yeastTimes, \
zip(data,scipy.ones_like(data)*constErrorBar )) )
#if usePreloadedData:
# save data for future use via usePreloadedData
simplePickle.save(yeastDict,yeastSavedDataFile)
#Plotting.plot(yeastTimes,v1data,'o-',label=str(temperature))
if (not usePreloadedData) and stopAfterRunningMATLAB: die
fittingData = [ yeastOscillatorData[d] for d in inputDescriptors ]
if useDerivs:
fittingDataDerivs = [ yeastOscillatorDataDerivs[d] for d in inputDescriptors ]
else:
fittingDataDerivs = None
return fittingData,fittingDataDerivs,inputVars,inputList
def setLock(fileNumString):
save(1, fileNumString + '_fileLocked.dat')
times, yeastRawData, params = simulateYeastOscillator([0, time], T, initialConditions=initialConditions)
speciesData.append(yeastRawData[:7, -1])
derivsData.append(yeastRawData[-7:, -1])
else: # 7.25.2012 keep all timepoints
times, yeastRawData, params = simulateYeastOscillator(
[0, maxTime], T, initialConditions=initialConditions
)
speciesData.extend(scipy.transpose(yeastRawData[:7]))
derivsData.extend(scipy.transpose(yeastRawData[-7:]))
if True:
speciesData = (scipy.transpose(speciesData)[:numSpecies]).T
derivsData = (scipy.transpose(derivsData)[:numSpecies]).T
else:
speciesData = speciesData[:numSpecies]
derivsData = derivsData[:numSpecies]
save((speciesData, derivsData), dataFilename)
# 7.19.2012 try using all data from single IC
# speciesData = yeastRawData[:numSpecies]
# derivsData = yeastRawData[-7:(-7+numSpecies)]
offset = 10 # 8.22.2012 staying away from zero is better?
indepParamsList = ICsList + offset
speciesData = speciesData + 10
# (one data point for each IC)
sigmas = 0.1 * scipy.ones_like(speciesData)
fittingData = [
dict(zip(visibleSpeciesNames, [dict([(randTime, (data, sigma))]) for data, sigma in zip(dataRow, sigmaRow)]))
for randTime, dataRow, sigmaRow in zip(randTimes, speciesData, sigmas)
]
def initializeFitAllParallel(fullFittingProblem,
fileNumString,
deltaNumDatapoints=2,
maxTimesPerIndepParam=None,
timeOrderSeed=123,
verbose=True,
numIndepParams=None):
"""
Creates data structure on disk for keeping track of fitting over increasing
amounts of data and multiple conditions.
After initialization, use runFitAllParallelWorker to run fitting.
Multiple workers can be run at the same time.
By default, add one timepoint for each independent parameter first,
then increase the number of timepoints per independent parameter.
Timepoints are added randomly for each independent parameter.
Independent parameters are added in the order of indepParamsList.
The amount of data is always kept equal across each condition.
If the length of indepParamsList or the number of timepoints per
independent parameter varies in the original, the total amount of
data used will be (#conditions)x(minimum number of indepParams per
condition)x(minimum number of timepoints per indepParam)
(that is, NOT ALL DATA WILL BE USED).
fullFittingProblem can be an instance of a FittingProblem or a
FittingProblemMultipleCondition.
deltaNumDatapoints (2) : The change in the number of datapoints
(per condition) between successive fits.
maxTimesPerIndepParam (None): The maximum number of timepoints used
per independent parameter.
numIndepParams (None) : Number of independent parameter
combinations (trials) to use in-sample per
condition. Defaults to the minimum
number available over conditions.
"""
# (only one fittingProblem if there are not multiple conditions)
fittingProblemList = getattr(fullFittingProblem, 'fittingProblemList',
[fullFittingProblem])
if fullFittingProblem.saveFilename is not None:
configString = fullFittingProblem.saveFilename[4:-4]
else:
configString = ''
# The length of fittingProblemList[0].fittingData is len(indepParamsList).
# N is the number of datapoints per condition.
# calculate maxN, the total number of datapoints per condition
numIndepParamsList, numTimepointsList = [], []
for fittingProblem in fittingProblemList:
numIndepParamsList.append(len(fittingProblem.fittingData))
for d in fittingProblem.fittingData:
numTimepointsList.append(len(d.values()[0]))
if numIndepParams is None:
numIndepParams = min(numIndepParamsList)
elif numIndepParams > min(numIndepParamsList):
raise Exception
minNumTimepoints = min(numTimepointsList)
if maxTimesPerIndepParam is not None:
minNumTimepoints = min(minNumTimepoints, maxTimesPerIndepParam)
maxN = numIndepParams * minNumTimepoints
Nlist = range(deltaNumDatapoints, maxN, deltaNumDatapoints)
Nlist = Nlist + [maxN]
createDirectoryStructure(fileNumString, len(fittingProblemList), Nlist)
# () With each increasing amount of data, make a copy of the fullFittingProblem
# that includes only that data.
fitProbData = {}
for N in Nlist:
fitProbDataList = []
for i, fittingProblem in enumerate(fittingProblemList):
fittingData = fittingProblem.fittingData
fittingDataSubset = dataSubset(fittingData,
N,
seed=timeOrderSeed + i,
maxNumIndepParams=numIndepParams)
indepParamsListSubset = \
fittingProblem.indepParamsList[:len(fittingDataSubset)]
newFittingProblem = copy.deepcopy(fittingProblem)
newFittingProblem.setData(fittingDataSubset, indepParamsListSubset,
newFittingProblem.indepParamNames)
newFittingProblem.saveKey = N
#fittingProblemListNew.append(newFittingProblem)
# store each full fittingProblem in separate file
fitProbDict = {N: newFittingProblem}
dirPrefix = directoryPrefix(fileNumString, i, N)
save(fitProbDict,
dirPrefix + fileNumString + configString + '.dat')
# in fitProbData, store only info necessary to decide which
# fittingProblem to work on next
fitProb = newFittingProblem
fittingStateDictInitial = \
dict( [ (name,'unstarted') for name in fitProb.fittingModelNames ])
pData = {
'logLikelihoodDict': fitProb.logLikelihoodDict,
'fittingStateDict': fittingStateDictInitial,
'fittingModelNames': fitProb.fittingModelNames,
'stopFittingN': fitProb.stopFittingN,
'saveFilename': fitProb.saveFilename,
'saveKey': N,
}
fitProbDataList.append(pData)
p = fullFittingProblem
cp = copy.deepcopy
pDataMultiple = {
'logLikelihoodDict': cp(p.logLikelihoodDict),
'fitAllDone': cp(p.fitAllDone),
'fittingModelNames': cp(p.fittingModelNames),
'fitProbDataList': cp(fitProbDataList),
'stopFittingN': cp(p.stopFittingN),
'saveFilename': cp(p.saveFilename),
'saveKey': cp(p.saveKey),
}
fitProbData[N] = pDataMultiple
save(fitProbData, fileNumString + '_fitProbData.dat')
if verbose:
print "initializeFitAllParallel: Done initializing N =", N
def saveFitProbData(fitProbData, fileNumString):
try:
save(fitProbData, fileNumString + '_fitProbData.dat')
except IOError:
print "saveFitProbData: WARNING Unable to save fitProbData file."
def saveFitProb(fitProb, saveFilename, fileNumString, conditioni,
numTimepoints):
dirPrefix = directoryPrefix(fileNumString, conditioni, numTimepoints)
fitProbDict = {numTimepoints: fitProb}
save(fitProbDict, dirPrefix + saveFilename)
def combineFitProbs(fileNumString,
saveCombined=True,
combinedLean=True,
reset=False):
"""
Combine fittingProblems from multiple conditions saved in the
parallel file structure into a single fittingProblemDict.
Currently only includes data from models that have been fit to
all conditions.
saveCombined (True) : Overwrites any current top-level fitProbDict file
with a combined fitProbDict containing all
numTimepoints. Set to False to minimize memory use.
combinedLean (True) : Combined fpd is saved without models to save memory.
reset (False) : If True, overwrite or delete any existing combined
fitProbDicts. This erases any existing
outOfSampleCost information.
"""
fitProbData = loadFitProbData(fileNumString)
saveFilename = fitProbData.values()[0]['saveFilename']
#save({},saveFilename)
if saveCombined: fpdMultiple = {}
fitSubsets = subsetsWithFits(fileNumString)
subsetsToCombine = subsetsWithFits(fileNumString, onlyNew=not reset)
for numTimepoints in fitSubsets:
Nfilename = directoryPrefixNonly(fileNumString,
numTimepoints) + '/' + saveFilename
fileExists = os.path.exists(Nfilename)
if fileExists and reset:
# then an old combined file exists -- erase it to reset
os.remove(Nfilename)
fileExists = False
print "combineFitProbs: Reset removed file for numTimepoints =", numTimepoints
if numTimepoints in subsetsToCombine: # combine
oldOutOfSampleCostDict = {}
if fileExists: # grab any out-of-sample cost data
fpMultiple = load(Nfilename)
if hasattr(fpMultiple, 'outOfSampleCostDict'):
oldOutOfSampleCostDict = fpMultiple.outOfSampleCostDict
p = fitProbData[numTimepoints]
fpList = []
for conditioni in range(len(p['fitProbDataList'])):
fp = loadFitProb(saveFilename, fileNumString, conditioni,
numTimepoints)
fpList.append(fp)
# make new multiple condition fitting problem by starting
# with an empty fitting problem and inserting the fittingProblemList
saveKey = p['saveKey']
fp.stopFittingN = p['stopFittingN']
fpMultiple = FittingProblemMultipleCondition([], [],
saveFilename=None,
saveKey=saveKey,
fp0=fp)
fpMultiple.fittingProblemList = fpList
fpMultiple.outOfSampleCostDict = oldOutOfSampleCostDict
# Populate the logLikelihoodDict, etc by running fitAll.
fpMultiple.fitAll(onlyCombine=True)
if saveCombined: fpdMultiple[numTimepoints] = fpMultiple
save(fpMultiple, Nfilename)
print "combineFitProbs: Done with numTimepoints =", numTimepoints
else: # no new fits to combine; just load from file
if saveCombined: fpdMultiple[numTimepoints] = load(Nfilename)
print "combineFitProbs: Done with numTimepoints =", numTimepoints
if saveCombined:
if combinedLean:
makeFpdLean(fpdMultiple)
save(fpdMultiple, saveFilename[:-4] + '_combined.dat')
# Get results from remaining worker processes
while num_completed < work_size: #-1
results, status = pypar.receive(source=pypar.any_source, tag=pypar.any_tag, return_status=True)
# save results
result_index = results.pop()
allOutputsDict[result_index] = results
num_completed += 1
# Shut down worker processes
for proc in range(1, num_processors):
print "Stopping worker process " + str(proc)
pypar.send(-1, proc, tag=DIE_TAG)
# Write data to file
save(allOutputsDict,outputFilename)
else:
### Worker Processes ###
continue_working = True
while continue_working:
work_index, status = pypar.receive( \
source=MASTER_PROCESS, tag=pypar.any_tag, \
return_status=True)
if status.tag == DIE_TAG:
continue_working = False
else:
work_array, status = pypar.receive( \
source=MASTER_PROCESS, tag=pypar.any_tag, \
# Get results from remaining worker processes
while num_completed < work_size: #-1
results, status = pypar.receive(source=pypar.any_source, tag=pypar.any_tag, return_status=True)
# save results
result_index = results.pop()
allOutputsDict[result_index] = results
num_completed += 1
# Shut down worker processes
for proc in range(1, num_processors):
print "Stopping worker process " + str(proc)
pypar.send(-1, proc, tag=DIE_TAG)
# Write data to file
save(allOutputsDict,outputFilename)
else:
### Worker Processes ###
continue_working = True
while continue_working:
work_index, status = pypar.receive( \
source=MASTER_PROCESS, tag=pypar.any_tag, \
return_status=True)
if status.tag == DIE_TAG:
continue_working = False
else:
work_array, status = pypar.receive( \
source=MASTER_PROCESS, tag=pypar.any_tag, \
},
'file_data': file_data
}),
dest=worker)
# get results
for worker in range(1, num_procs):
msg = comm.recv(source=worker)
print("mpi_test: Worker {} said {}".format(worker, msg))
file_data['test_output'] = msg
# stop workers
for worker in range(1, num_procs):
comm.send(SystemExit(), dest=worker)
save(file_data, file_data['output_filename'])
while my_rank != 0:
# worker process
# Wait for a message
message = comm.recv(source=0)
if isinstance(message, SystemExit):
sys.exit()
command, msg_locals = message
locals().update(msg_locals)
result = eval(command)
comm.send(result, dest=0)
