times,yeastRawData,params = simulateYeastOscillator([0,time],T, \ initialConditions=initialConditions) speciesData.append(yeastRawData[:7, -1]) derivsData.append(yeastRawData[-7:, -1]) else: # 7.25.2012 keep all timepoints times,yeastRawData,params = simulateYeastOscillator([0,maxTime],T, \ initialConditions=initialConditions) speciesData.extend(scipy.transpose(yeastRawData[:7])) derivsData.extend(scipy.transpose(yeastRawData[-7:])) if True: speciesData = (scipy.transpose(speciesData)[:numSpecies]).T derivsData = (scipy.transpose(derivsData)[:numSpecies]).T else: speciesData = speciesData[:numSpecies] derivsData = derivsData[:numSpecies] save((speciesData, derivsData), dataFilename) # 7.19.2012 try using all data from single IC #speciesData = yeastRawData[:numSpecies] #derivsData = yeastRawData[-7:(-7+numSpecies)] offset = 10 # 8.22.2012 staying away from zero is better? indepParamsList = ICsList + offset speciesData = speciesData + 10 # (one data point for each IC) sigmas = 0.1 * scipy.ones_like(speciesData) fittingData = [ dict( zip( visibleSpeciesNames, \ [ dict( [(randTime,(data,sigma))] ) \ for data,sigma in zip(dataRow,sigmaRow)] ) ) \ for randTime,dataRow,sigmaRow in zip(randTimes,speciesData,sigmas) ]
def yeastData(numPoints,timeInterval, \ numICs,useDerivs=True,includedIndices=range(7), \ timesSeed=3,noiseSeed=4,ICseed=5, \ multiplicativeErrorBar=0.1,upperRangeMultiple=1., \ randomX=True,inputVars=None,nonzeroMin=True, \ yeastSavedDataFile=None,includeNoise=True): """ upperRangeMultiple (1.) : Each range of initial conditions is expanded by this factor by increasing the upper limit. nonzeroMin (True) : Use the lower limit for initial conditions given in SchValJen11; otherwise use 0. as lower limit. """ usePreloadedData = True #<<<<<<<******************* allNames = scipy.array(['S1', 'S2', 'S3', 'S4', 'N2', 'A3', 'S4ex']) names = allNames[includedIndices] varList = names defaultTemperature = 286.5 #changed 10.16.2013 (was 288 K before) # Table 2 of RuoChrWol03 defaultICs = scipy.array([1.187, 0.193, 0.050, 0.115, 0.077, 2.475, 0.077]) # mM # choose input variable(s) if False: # use temp. as input inputVars = ['Temperature'] inputList = [[278], [288], [293]] inputDescriptors = [inputs[0] for inputs in inputList] elif False: # only a single temperature value inputVars = ['Temperature'] inputList = [[defaultTemperature]] inputDescriptors = [inputs[0] for inputs in inputList] elif inputVars is None: # use varying initial conditions on all 7 species inputVars = [name + "_init" for name in names] print "inputVars =", inputVars # taken from SchValJen11 Table 2 ICranges = scipy.array( \ [[0.15,1.60],[0.19,2.16], \ [0.04,0.20],[0.10,0.35], \ [0.08,0.30],[0.14,2.67],[0.05,0.10]] )[includedIndices] # mM # as I vary the number of ICs, I want the same sequence of random ICs scipy.random.seed(ICseed) randICmults = [] for i in range(numICs): randICmults.append(scipy.rand(len(includedIndices))) #randomICs = scipy.rand(numICs,len(includedIndices))* \ # (ICranges[:,1]-ICranges[:,0]) + ICranges[:,0] randomICs = scipy.array(randICmults)*upperRangeMultiple* \ (ICranges[:,1]-nonzeroMin*ICranges[:,0]) + nonzeroMin*ICranges[:,0] inputList = randomICs inputDescriptors = range(len(inputList)) else: raise Exception, "Changing inputVars not yet implemented" # as I vary the number of ICs, I want the same sequence of lists of times scipy.random.seed(timesSeed) timesRandList = [] for i in range(numICs): timesRandList.append(scipy.rand(numPoints)) # all other random stuff is done now except for the noise # as I vary the number of ICs, I want the same sequence of noise scipy.random.seed(noiseSeed) # set up preloaded data dict if yeastSavedDataFile is None: yeastSavedDataFile = "yeastExampleData.fittingData" yeastDict = {} if usePreloadedData: try: #yeastTimes,loadedICs,yeastRawData = \ # simplePickle.load(yeastDataFilename) yeastDict = simplePickle.load(yeastSavedDataFile) except IOError: print "yeastData: Could not load " + yeastSavedDataFile usePreloadedData = False yeastOscillatorData = {} yeastOscillatorDataDerivs = {} for inputs, inputDescriptor, timesRand in zip(inputList, inputDescriptors, timesRandList): stopAfterRunningMATLAB = False # deal with independent parameters # if it's temperature (only) if inputVars is ['Temperature']: temperature = inputs[0] else: temperature = defaultTemperature initialConditions = copy.deepcopy(defaultICs) for inputVar, input in zip(inputVars, inputs): # if it's initial conditions if inputVar[-5:] == "_init": index = pylab.find(allNames == inputVar[:-5])[0] initialConditions[index] = input print "initialConditions =", initialConditions eps = (timeInterval[1] - timeInterval[0] ) / 1000. #1e-3 #1e-4 # (minutes) resolution of timepoints integratorTimes = scipy.arange(timeInterval[0], timeInterval[1] + eps, eps) if not randomX: desiredTimes = scipy.linspace(timeInterval[0], timeInterval[1], numPoints) else: desiredTimes = timesRand * (timeInterval[1]-timeInterval[0]) \ + timeInterval[0] yeastDataKey = copy.deepcopy( \ (tuple(initialConditions),inputDescriptor,tuple(desiredTimes))) runMATLAB = not usePreloadedData if usePreloadedData: if yeastDict.has_key(yeastDataKey): yeastTimes, loadedICs, yeastRawData = yeastDict[yeastDataKey] if not scipy.all(loadedICs == initialConditions): print "loadedICs =", loadedICs print "initialConditions =", initialConditions raise Exception, "loadedICs != initialConditions" else: print "yeastData: "+yeastSavedDataFile+" does not " \ "contain the necessary data." runMATLAB = True if runMATLAB: print "yeastData: Running simulateYeastOscillator." yeastTimes,yeastRawData,yeastParams = \ simulateYeastOscillator(integratorTimes,temperature, \ initialConditions=initialConditions) # the integrator returns more timepoints than I want desiredIndices = [ pylab.find(abs(yeastTimes-time)<eps/2)[0] \ for time in desiredTimes ] yeastRawData = yeastRawData[:, desiredIndices] yeastTimes = yeastTimes[:, desiredIndices] yeastDict[yeastDataKey] = (yeastTimes, initialConditions, yeastRawData) if False: # OLD! use temp. as input, calculate v1 as output, mult. error bars S1data = yeastRawData[0] A3data = yeastRawData[5] k1, K1, q = yeastParams[1], yeastParams[10], yeastParams[9] v1data = k1 * S1data * A3data / (1. + (A3data / K1)**q) yeastOscillatorData[inputDescriptor] = {} yeastOscillatorData[inputDescriptor]['v1'] = \ dict( zip(times, zip(v1data,multiplicativeErrorBar*v1data)) ) varList = ['v1'] if useDerivs: # use derivatives as output (7-dimensional), const. error bars # 2.22.2012 from Table 2 of SchValJen11 # 11.29.2012 using these even though they are stddevs of values, not derivs stddevs = [0.4872, 0.6263, 0.0503, 0.0814, 0.0379, 0.7478, 0.0159] # mM stddevs = scipy.array(stddevs)[includedIndices] #derivNames = [ 'ddt_'+name for name in names ] derivDataList = yeastRawData[7:][includedIndices] yeastOscillatorDataDerivs[inputDescriptor] = {} for derivName, derivData, stddev in zip(names, derivDataList, stddevs): constErrorBar = multiplicativeErrorBar * stddev yeastOscillatorDataDerivs[inputDescriptor][derivName] = \ dict( zip(yeastTimes, \ zip(derivData,scipy.ones_like(derivData)*constErrorBar)) ) #varList = names # 10.3.2011 if True: # use all species as output (7-dimensional), const. error bars # 2.22.2012 from Table 2 of SchValJen11 stddevs = [0.4872, 0.6263, 0.0503, 0.0814, 0.0379, 0.7478, 0.0159] # mM #includeNoise = False #True # <<<<****** 12.5.2012 to avoid negative values verbose = True dataList = scipy.array(yeastRawData)[:7][includedIndices] stddevs = scipy.array(stddevs)[includedIndices] # 11.17.2011 reorder stuff #names,dataList = names[::-1],dataList[::-1] yeastOscillatorData[inputDescriptor] = {} for name, data, stddev in zip(names, dataList, stddevs): #constErrorBar = multiplicativeErrorBar*scipy.mean(data) #*data[0] constErrorBar = multiplicativeErrorBar * stddev if includeNoise: noise = scipy.random.normal(scale=constErrorBar, size=len(data)) data = data + noise yeastOscillatorData[inputDescriptor][name] = \ dict( zip(yeastTimes, \ zip(data,scipy.ones_like(data)*constErrorBar )) ) #if usePreloadedData: # save data for future use via usePreloadedData simplePickle.save(yeastDict, yeastSavedDataFile) #Plotting.plot(yeastTimes,v1data,'o-',label=str(temperature)) if (not usePreloadedData) and stopAfterRunningMATLAB: die fittingData = [yeastOscillatorData[d] for d in inputDescriptors] if useDerivs: fittingDataDerivs = [ yeastOscillatorDataDerivs[d] for d in inputDescriptors ] else: fittingDataDerivs = None return fittingData, fittingDataDerivs, inputVars, inputList
def yeastData(numPoints,timeInterval, \ numICs,useDerivs=True,includedIndices=range(7), \ timesSeed=3,noiseSeed=4,ICseed=5, \ multiplicativeErrorBar=0.1,upperRangeMultiple=1., \ randomX=True,inputVars=None,nonzeroMin=True, \ yeastSavedDataFile=None,includeNoise=True): """ upperRangeMultiple (1.) : Each range of initial conditions is expanded by this factor by increasing the upper limit. nonzeroMin (True) : Use the lower limit for initial conditions given in SchValJen11; otherwise use 0. as lower limit. """ usePreloadedData = True #<<<<<<<******************* allNames = scipy.array(['S1','S2','S3','S4','N2','A3','S4ex']) names = allNames[includedIndices] varList = names defaultTemperature = 286.5 #changed 10.16.2013 (was 288 K before) # Table 2 of RuoChrWol03 defaultICs = scipy.array([1.187,0.193,0.050,0.115,0.077,2.475,0.077]) # mM # choose input variable(s) if False: # use temp. as input inputVars = ['Temperature'] inputList = [[278],[288],[293]] inputDescriptors = [inputs[0] for inputs in inputList] elif False: # only a single temperature value inputVars = ['Temperature'] inputList = [[defaultTemperature]] inputDescriptors = [inputs[0] for inputs in inputList] elif inputVars is None: # use varying initial conditions on all 7 species inputVars = [name+"_init" for name in names] print "inputVars =",inputVars # taken from SchValJen11 Table 2 ICranges = scipy.array( \ [[0.15,1.60],[0.19,2.16], \ [0.04,0.20],[0.10,0.35], \ [0.08,0.30],[0.14,2.67],[0.05,0.10]] )[includedIndices] # mM # as I vary the number of ICs, I want the same sequence of random ICs scipy.random.seed(ICseed) randICmults = [] for i in range(numICs): randICmults.append(scipy.rand(len(includedIndices))) #randomICs = scipy.rand(numICs,len(includedIndices))* \ # (ICranges[:,1]-ICranges[:,0]) + ICranges[:,0] randomICs = scipy.array(randICmults)*upperRangeMultiple* \ (ICranges[:,1]-nonzeroMin*ICranges[:,0]) + nonzeroMin*ICranges[:,0] inputList = randomICs inputDescriptors = range(len(inputList)) else: raise Exception, "Changing inputVars not yet implemented" # as I vary the number of ICs, I want the same sequence of lists of times scipy.random.seed(timesSeed) timesRandList = [] for i in range(numICs): timesRandList.append(scipy.rand(numPoints)) # all other random stuff is done now except for the noise # as I vary the number of ICs, I want the same sequence of noise scipy.random.seed(noiseSeed) # set up preloaded data dict if yeastSavedDataFile is None: yeastSavedDataFile = "yeastExampleData.fittingData" yeastDict = {} if usePreloadedData: try: #yeastTimes,loadedICs,yeastRawData = \ # simplePickle.load(yeastDataFilename) yeastDict = simplePickle.load(yeastSavedDataFile) except IOError: print "yeastData: Could not load "+yeastSavedDataFile usePreloadedData = False yeastOscillatorData = {} yeastOscillatorDataDerivs = {} for inputs,inputDescriptor,timesRand in zip(inputList,inputDescriptors,timesRandList): stopAfterRunningMATLAB = False # deal with independent parameters # if it's temperature (only) if inputVars is ['Temperature']: temperature = inputs[0] else: temperature = defaultTemperature initialConditions = copy.deepcopy(defaultICs) for inputVar,input in zip(inputVars,inputs): # if it's initial conditions if inputVar[-5:] == "_init": index = pylab.find(allNames==inputVar[:-5])[0] initialConditions[index] = input print "initialConditions =",initialConditions eps = (timeInterval[1]-timeInterval[0])/1000. #1e-3 #1e-4 # (minutes) resolution of timepoints integratorTimes = scipy.arange(timeInterval[0],timeInterval[1]+eps,eps) if not randomX: desiredTimes = scipy.linspace(timeInterval[0],timeInterval[1],numPoints) else: desiredTimes = timesRand * (timeInterval[1]-timeInterval[0]) \ + timeInterval[0] yeastDataKey = copy.deepcopy( \ (tuple(initialConditions),inputDescriptor,tuple(desiredTimes))) runMATLAB = not usePreloadedData if usePreloadedData: if yeastDict.has_key(yeastDataKey): yeastTimes,loadedICs,yeastRawData = yeastDict[yeastDataKey] if not scipy.all(loadedICs == initialConditions): print "loadedICs =",loadedICs print "initialConditions =",initialConditions raise Exception, "loadedICs != initialConditions" else: print "yeastData: "+yeastSavedDataFile+" does not " \ "contain the necessary data." runMATLAB = True if runMATLAB: print "yeastData: Running simulateYeastOscillator." yeastTimes,yeastRawData,yeastParams = \ simulateYeastOscillator(integratorTimes,temperature, \ initialConditions=initialConditions) # the integrator returns more timepoints than I want desiredIndices = [ pylab.find(abs(yeastTimes-time)<eps/2)[0] \ for time in desiredTimes ] yeastRawData = yeastRawData[:,desiredIndices] yeastTimes = yeastTimes[:,desiredIndices] yeastDict[yeastDataKey] = (yeastTimes,initialConditions,yeastRawData) if False: # OLD! use temp. as input, calculate v1 as output, mult. error bars S1data = yeastRawData[0] A3data = yeastRawData[5] k1,K1,q = yeastParams[1],yeastParams[10],yeastParams[9] v1data = k1*S1data*A3data/(1.+(A3data/K1)**q) yeastOscillatorData[inputDescriptor] = {} yeastOscillatorData[inputDescriptor]['v1'] = \ dict( zip(times, zip(v1data,multiplicativeErrorBar*v1data)) ) varList = ['v1'] if useDerivs: # use derivatives as output (7-dimensional), const. error bars # 2.22.2012 from Table 2 of SchValJen11 # 11.29.2012 using these even though they are stddevs of values, not derivs stddevs = [0.4872,0.6263,0.0503,0.0814,0.0379,0.7478,0.0159] # mM stddevs = scipy.array(stddevs)[includedIndices] #derivNames = [ 'ddt_'+name for name in names ] derivDataList = yeastRawData[7:][includedIndices] yeastOscillatorDataDerivs[inputDescriptor] = {} for derivName,derivData,stddev in zip(names,derivDataList,stddevs): constErrorBar = multiplicativeErrorBar*stddev yeastOscillatorDataDerivs[inputDescriptor][derivName] = \ dict( zip(yeastTimes, \ zip(derivData,scipy.ones_like(derivData)*constErrorBar)) ) #varList = names # 10.3.2011 if True: # use all species as output (7-dimensional), const. error bars # 2.22.2012 from Table 2 of SchValJen11 stddevs = [0.4872,0.6263,0.0503,0.0814,0.0379,0.7478,0.0159] # mM #includeNoise = False #True # <<<<****** 12.5.2012 to avoid negative values verbose = True dataList = scipy.array(yeastRawData)[:7][includedIndices] stddevs = scipy.array(stddevs)[includedIndices] # 11.17.2011 reorder stuff #names,dataList = names[::-1],dataList[::-1] yeastOscillatorData[inputDescriptor] = {} for name,data,stddev in zip(names,dataList,stddevs): #constErrorBar = multiplicativeErrorBar*scipy.mean(data) #*data[0] constErrorBar = multiplicativeErrorBar*stddev if includeNoise: noise = scipy.random.normal(scale=constErrorBar,size=len(data)) data = data + noise yeastOscillatorData[inputDescriptor][name] = \ dict( zip(yeastTimes, \ zip(data,scipy.ones_like(data)*constErrorBar )) ) #if usePreloadedData: # save data for future use via usePreloadedData simplePickle.save(yeastDict,yeastSavedDataFile) #Plotting.plot(yeastTimes,v1data,'o-',label=str(temperature)) if (not usePreloadedData) and stopAfterRunningMATLAB: die fittingData = [ yeastOscillatorData[d] for d in inputDescriptors ] if useDerivs: fittingDataDerivs = [ yeastOscillatorDataDerivs[d] for d in inputDescriptors ] else: fittingDataDerivs = None return fittingData,fittingDataDerivs,inputVars,inputList
def setLock(fileNumString): save(1, fileNumString + '_fileLocked.dat')
times, yeastRawData, params = simulateYeastOscillator([0, time], T, initialConditions=initialConditions) speciesData.append(yeastRawData[:7, -1]) derivsData.append(yeastRawData[-7:, -1]) else: # 7.25.2012 keep all timepoints times, yeastRawData, params = simulateYeastOscillator( [0, maxTime], T, initialConditions=initialConditions ) speciesData.extend(scipy.transpose(yeastRawData[:7])) derivsData.extend(scipy.transpose(yeastRawData[-7:])) if True: speciesData = (scipy.transpose(speciesData)[:numSpecies]).T derivsData = (scipy.transpose(derivsData)[:numSpecies]).T else: speciesData = speciesData[:numSpecies] derivsData = derivsData[:numSpecies] save((speciesData, derivsData), dataFilename) # 7.19.2012 try using all data from single IC # speciesData = yeastRawData[:numSpecies] # derivsData = yeastRawData[-7:(-7+numSpecies)] offset = 10 # 8.22.2012 staying away from zero is better? indepParamsList = ICsList + offset speciesData = speciesData + 10 # (one data point for each IC) sigmas = 0.1 * scipy.ones_like(speciesData) fittingData = [ dict(zip(visibleSpeciesNames, [dict([(randTime, (data, sigma))]) for data, sigma in zip(dataRow, sigmaRow)])) for randTime, dataRow, sigmaRow in zip(randTimes, speciesData, sigmas) ]
def initializeFitAllParallel(fullFittingProblem, fileNumString, deltaNumDatapoints=2, maxTimesPerIndepParam=None, timeOrderSeed=123, verbose=True, numIndepParams=None): """ Creates data structure on disk for keeping track of fitting over increasing amounts of data and multiple conditions. After initialization, use runFitAllParallelWorker to run fitting. Multiple workers can be run at the same time. By default, add one timepoint for each independent parameter first, then increase the number of timepoints per independent parameter. Timepoints are added randomly for each independent parameter. Independent parameters are added in the order of indepParamsList. The amount of data is always kept equal across each condition. If the length of indepParamsList or the number of timepoints per independent parameter varies in the original, the total amount of data used will be (#conditions)x(minimum number of indepParams per condition)x(minimum number of timepoints per indepParam) (that is, NOT ALL DATA WILL BE USED). fullFittingProblem can be an instance of a FittingProblem or a FittingProblemMultipleCondition. deltaNumDatapoints (2) : The change in the number of datapoints (per condition) between successive fits. maxTimesPerIndepParam (None): The maximum number of timepoints used per independent parameter. numIndepParams (None) : Number of independent parameter combinations (trials) to use in-sample per condition. Defaults to the minimum number available over conditions. """ # (only one fittingProblem if there are not multiple conditions) fittingProblemList = getattr(fullFittingProblem, 'fittingProblemList', [fullFittingProblem]) if fullFittingProblem.saveFilename is not None: configString = fullFittingProblem.saveFilename[4:-4] else: configString = '' # The length of fittingProblemList[0].fittingData is len(indepParamsList). # N is the number of datapoints per condition. # calculate maxN, the total number of datapoints per condition numIndepParamsList, numTimepointsList = [], [] for fittingProblem in fittingProblemList: numIndepParamsList.append(len(fittingProblem.fittingData)) for d in fittingProblem.fittingData: numTimepointsList.append(len(d.values()[0])) if numIndepParams is None: numIndepParams = min(numIndepParamsList) elif numIndepParams > min(numIndepParamsList): raise Exception minNumTimepoints = min(numTimepointsList) if maxTimesPerIndepParam is not None: minNumTimepoints = min(minNumTimepoints, maxTimesPerIndepParam) maxN = numIndepParams * minNumTimepoints Nlist = range(deltaNumDatapoints, maxN, deltaNumDatapoints) Nlist = Nlist + [maxN] createDirectoryStructure(fileNumString, len(fittingProblemList), Nlist) # () With each increasing amount of data, make a copy of the fullFittingProblem # that includes only that data. fitProbData = {} for N in Nlist: fitProbDataList = [] for i, fittingProblem in enumerate(fittingProblemList): fittingData = fittingProblem.fittingData fittingDataSubset = dataSubset(fittingData, N, seed=timeOrderSeed + i, maxNumIndepParams=numIndepParams) indepParamsListSubset = \ fittingProblem.indepParamsList[:len(fittingDataSubset)] newFittingProblem = copy.deepcopy(fittingProblem) newFittingProblem.setData(fittingDataSubset, indepParamsListSubset, newFittingProblem.indepParamNames) newFittingProblem.saveKey = N #fittingProblemListNew.append(newFittingProblem) # store each full fittingProblem in separate file fitProbDict = {N: newFittingProblem} dirPrefix = directoryPrefix(fileNumString, i, N) save(fitProbDict, dirPrefix + fileNumString + configString + '.dat') # in fitProbData, store only info necessary to decide which # fittingProblem to work on next fitProb = newFittingProblem fittingStateDictInitial = \ dict( [ (name,'unstarted') for name in fitProb.fittingModelNames ]) pData = { 'logLikelihoodDict': fitProb.logLikelihoodDict, 'fittingStateDict': fittingStateDictInitial, 'fittingModelNames': fitProb.fittingModelNames, 'stopFittingN': fitProb.stopFittingN, 'saveFilename': fitProb.saveFilename, 'saveKey': N, } fitProbDataList.append(pData) p = fullFittingProblem cp = copy.deepcopy pDataMultiple = { 'logLikelihoodDict': cp(p.logLikelihoodDict), 'fitAllDone': cp(p.fitAllDone), 'fittingModelNames': cp(p.fittingModelNames), 'fitProbDataList': cp(fitProbDataList), 'stopFittingN': cp(p.stopFittingN), 'saveFilename': cp(p.saveFilename), 'saveKey': cp(p.saveKey), } fitProbData[N] = pDataMultiple save(fitProbData, fileNumString + '_fitProbData.dat') if verbose: print "initializeFitAllParallel: Done initializing N =", N
def saveFitProbData(fitProbData, fileNumString): try: save(fitProbData, fileNumString + '_fitProbData.dat') except IOError: print "saveFitProbData: WARNING Unable to save fitProbData file."
def saveFitProb(fitProb, saveFilename, fileNumString, conditioni, numTimepoints): dirPrefix = directoryPrefix(fileNumString, conditioni, numTimepoints) fitProbDict = {numTimepoints: fitProb} save(fitProbDict, dirPrefix + saveFilename)
def combineFitProbs(fileNumString, saveCombined=True, combinedLean=True, reset=False): """ Combine fittingProblems from multiple conditions saved in the parallel file structure into a single fittingProblemDict. Currently only includes data from models that have been fit to all conditions. saveCombined (True) : Overwrites any current top-level fitProbDict file with a combined fitProbDict containing all numTimepoints. Set to False to minimize memory use. combinedLean (True) : Combined fpd is saved without models to save memory. reset (False) : If True, overwrite or delete any existing combined fitProbDicts. This erases any existing outOfSampleCost information. """ fitProbData = loadFitProbData(fileNumString) saveFilename = fitProbData.values()[0]['saveFilename'] #save({},saveFilename) if saveCombined: fpdMultiple = {} fitSubsets = subsetsWithFits(fileNumString) subsetsToCombine = subsetsWithFits(fileNumString, onlyNew=not reset) for numTimepoints in fitSubsets: Nfilename = directoryPrefixNonly(fileNumString, numTimepoints) + '/' + saveFilename fileExists = os.path.exists(Nfilename) if fileExists and reset: # then an old combined file exists -- erase it to reset os.remove(Nfilename) fileExists = False print "combineFitProbs: Reset removed file for numTimepoints =", numTimepoints if numTimepoints in subsetsToCombine: # combine oldOutOfSampleCostDict = {} if fileExists: # grab any out-of-sample cost data fpMultiple = load(Nfilename) if hasattr(fpMultiple, 'outOfSampleCostDict'): oldOutOfSampleCostDict = fpMultiple.outOfSampleCostDict p = fitProbData[numTimepoints] fpList = [] for conditioni in range(len(p['fitProbDataList'])): fp = loadFitProb(saveFilename, fileNumString, conditioni, numTimepoints) fpList.append(fp) # make new multiple condition fitting problem by starting # with an empty fitting problem and inserting the fittingProblemList saveKey = p['saveKey'] fp.stopFittingN = p['stopFittingN'] fpMultiple = FittingProblemMultipleCondition([], [], saveFilename=None, saveKey=saveKey, fp0=fp) fpMultiple.fittingProblemList = fpList fpMultiple.outOfSampleCostDict = oldOutOfSampleCostDict # Populate the logLikelihoodDict, etc by running fitAll. fpMultiple.fitAll(onlyCombine=True) if saveCombined: fpdMultiple[numTimepoints] = fpMultiple save(fpMultiple, Nfilename) print "combineFitProbs: Done with numTimepoints =", numTimepoints else: # no new fits to combine; just load from file if saveCombined: fpdMultiple[numTimepoints] = load(Nfilename) print "combineFitProbs: Done with numTimepoints =", numTimepoints if saveCombined: if combinedLean: makeFpdLean(fpdMultiple) save(fpdMultiple, saveFilename[:-4] + '_combined.dat')
# Get results from remaining worker processes while num_completed < work_size: #-1 results, status = pypar.receive(source=pypar.any_source, tag=pypar.any_tag, return_status=True) # save results result_index = results.pop() allOutputsDict[result_index] = results num_completed += 1 # Shut down worker processes for proc in range(1, num_processors): print "Stopping worker process " + str(proc) pypar.send(-1, proc, tag=DIE_TAG) # Write data to file save(allOutputsDict,outputFilename) else: ### Worker Processes ### continue_working = True while continue_working: work_index, status = pypar.receive( \ source=MASTER_PROCESS, tag=pypar.any_tag, \ return_status=True) if status.tag == DIE_TAG: continue_working = False else: work_array, status = pypar.receive( \ source=MASTER_PROCESS, tag=pypar.any_tag, \
}, 'file_data': file_data }), dest=worker) # get results for worker in range(1, num_procs): msg = comm.recv(source=worker) print("mpi_test: Worker {} said {}".format(worker, msg)) file_data['test_output'] = msg # stop workers for worker in range(1, num_procs): comm.send(SystemExit(), dest=worker) save(file_data, file_data['output_filename']) while my_rank != 0: # worker process # Wait for a message message = comm.recv(source=0) if isinstance(message, SystemExit): sys.exit() command, msg_locals = message locals().update(msg_locals) result = eval(command) comm.send(result, dest=0)