def runBatchAsSingleProcess(batchDataList, quiet = False):
'''
Run a set of simulations on one core without multiprocessing
Args:
batchDataList (list) :
with data for each batch job [batchData1, batchData .. ]
batchData (dict) :
dict with {simulationIndex: , networkName: , dataNumber: , networkXmlFile: , pathSolutionDataFilename: }
'''
timeStartBatch = time.time()
print '====================================='
print '------Single Process Batch Job-------'
print 'numberOfEval.: {}'.format(len(batchDataList))
progressBar = cPB.ProgressBar(35, len(batchDataList))
for completed,batchData in enumerate(batchDataList):
minutesSolve,secsSolve = runSingleBatchSimulation(batchData)
if quiet == False:
print '____________Batch {:5} ___________'.format(batchDataList.index(batchData))
print 'Runtime: {} min {} sec'.format(minutesSolve,secsSolve)
progressBar.progress(completed)
timeBatchJob= time.time()-timeStartBatch
minutesBatch = int(timeBatchJob/60.)
secsBatch = timeBatchJob-minutesBatch*60.
print '====================================='
print 'total runtime: {} min {} sec'.format(minutesBatch,secsBatch)
print '====================================='
def createEvaluationCaseFiles(self):
'''
batchDataList <list> := with data for each batch job [batchData1, batchData .. ]
batchData <dict> := dict with {simulationIndex: , networkName: , dataNumber: , networkXmlFile: , pathSolutionDataFilename: }
'''
self.evaluationCaseFiles = [
] # list of dict: [ caseFileDict1,caseFileDict2 ..] for each evaluation
#TODO: replace create evaluation files or save them to disc!!!
if self.simulateEvaluations == True or self.preProcessData == True or self.createEvaluationXmlFiles == True:
sampleSize = self.sampleManager.currentSampleSize
print "Create evaluation case file list"
progressBar = cPB.ProgressBar(35, sampleSize)
for simulationIndex in xrange(sampleSize):
networkXmlFileLoad = mFPH_VPC.getFilePath(
'uqsaEvaluationNetworkXmlFile',
self.networkName,
self.dataNumber,
'write',
caseName=self.sampleManager.samplingMethod,
evaluationNumber=simulationIndex)
networkXmlFileSave = networkXmlFileLoad
pathSolutionDataFilename = mFPH_VPC.getFilePath(
'uqsaEvaluationSolutionDataFile',
self.networkName,
self.dataNumber,
'write',
caseName=self.sampleManager.samplingMethod,
evaluationNumber=simulationIndex)
caseFileDict1 = {
'simulationIndex': simulationIndex,
'networkName': self.networkName,
'dataNumber': self.dataNumber,
'networkXmlFileLoad': networkXmlFileLoad,
'networkXmlFileSave': networkXmlFileSave,
'pathSolutionDataFilename': pathSolutionDataFilename
}
self.evaluationCaseFiles.append(caseFileDict1)
progressBar.progress(simulationIndex)
def hashDataForGivenBases(self, basis, sampleSize):
'''
'''
self.hdf5Group.create_dataset('trajectoryData', (sampleSize,len(basis)) , dtype='float64')
self.hdf5Group.create_dataset('trajectoryBasis', data = basis, dtype='float64')
progressBar = cPB.ProgressBar(35, sampleSize)
for n in xrange(sampleSize):
data = self.hdf5Group['data'][n]
dataBasis = self.hdf5Group['dataBasis'][n]
self.hdf5Group['trajectoryData'][n] = np.interp(basis, dataBasis, data)
progressBar.progress(n)
def runBatchAsMultiprocessing(batchDataList, numberWorkers = None, quiet = False):
'''
Run a set of simulations on one core without multiprocessing
Args:
batchDataList (list) :
with data for each batch job [batchData1, batchData .. ]
batchData (dict) :
dict with {simulationIndex: , networkName: , dataNumber: , networkXmlFile: , pathSolutionDataFilename: }
'''
if numberWorkers == None: numberWorkers = multiprocessing.cpu_count()
timeStartBatch = time.time()
print '====================================='
print '------Multiprocessing Batch Job------'
print 'numberWorkers: {}'.format(numberWorkers)
print 'numberOfEval.: {}'.format(len(batchDataList))
progressBar = cPB.ProgressBar(35, len(batchDataList))
pool = multiprocessing.Pool(numberWorkers, maxtasksperchild = None)
results = pool.imap(runSingleBatchSimulation,batchDataList)
pool.close()
while (True):
completed = results._index
if (completed == len(batchDataList)): break
progressBar.progress(completed)
pool.join()
if quiet == False:
print '====================================='
for batchJobIndex,[minutesSolve,secsSolve] in enumerate(results):
print '____________Batch {:5} ___________'.format(batchJobIndex+1)
print 'Runtime: {} min {} sec'.format(minutesSolve,secsSolve)
print '====================================='
timeBatchJob= time.time()-timeStartBatch
minutesBatch = int(timeBatchJob/60.)
secsBatch = timeBatchJob-minutesBatch*60.
print 'total runtime: {} min {} sec'.format(minutesBatch,secsBatch)
print '====================================='
def preprocessSolutionData(self, evaluationCaseFiles,
preprocessedSolutionData,
simulationTimeFileSave, simulationTimeFileLoad):
'''
load all samples and pass data to locations of interest
find simulation time array with largest dt
invoke time cropping and post processing for the data
'''
# loop through all solution data files
# find time array
#solutionTime min
nPoints = 1e32
timeS = 0
timeE = 1e32
if self.evaluateSimulationTime == True:
print "estimate simulation time of all simulations"
progressBar = cPB.ProgressBar(35, len(evaluationCaseFiles))
for batchData in evaluationCaseFiles:
networkName = batchData['networkName']
dataNumber = batchData['dataNumber']
networkXmlFileLoad = batchData['networkXmlFileLoad']
pathSolutionDataFilename = batchData[
'pathSolutionDataFilename']
vascularNetwork = moduleXML.loadNetworkFromXML(
networkName,
dataNumber,
networkXmlFile=networkXmlFileLoad,
pathSolutionDataFilename=pathSolutionDataFilename)
vascularNetwork.linkSolutionData()
cPoints = len(vascularNetwork.simulationTime)
ctimeS = min(vascularNetwork.simulationTime)
ctimeE = max(vascularNetwork.simulationTime)
if cPoints < nPoints:
nPoints = cPoints
if ctimeS > timeS:
timeS = ctimeS
if ctimeE < timeE:
timeE = ctimeE
vascularNetwork.solutionDataFile.close()
del vascularNetwork
progressBar.progress(evaluationCaseFiles.index(batchData))
self.simulationTime = np.linspace(timeS, timeE, nPoints)
# save the simulationTime
self.saveQuantitiyOfInterestData(simulationTimeFileSave)
else:
print "load estimated simulation time of all simulations"
if simulationTimeFileLoad != None:
self.loadQuantitiyOfInterestData(simulationTimeFileLoad)
else:
raise ValueError(
'simulationTime hdf5 file for case does not exist! {}'.
format(simulationTimeFileLoad))
self.openQuantityOfInterestFile(preprocessedSolutionData)
print "estimate solution data for quantities of interest"
progressBar = cPB.ProgressBar(35, len(evaluationCaseFiles))
# pass the data to the locationsOfInterests which will load the information needed
for batchData in evaluationCaseFiles:
simulationIndex = batchData['simulationIndex']
networkName = batchData['networkName']
dataNumber = batchData['dataNumber']
networkXmlFileLoad = batchData['networkXmlFileLoad']
pathSolutionDataFilename = batchData['pathSolutionDataFilename']
vascularNetwork = moduleXML.loadNetworkFromXML(
networkName,
dataNumber,
networkXmlFile=networkXmlFileLoad,
pathSolutionDataFilename=pathSolutionDataFilename)
vascularNetwork.linkSolutionData()
for locationOfInterest in self.locationsOfInterest.values():
locationOfInterest.preprocessSolutionData(
vascularNetwork, self.simulationTime, self.sampleSize,
simulationIndex)
progressBar.progress(evaluationCaseFiles.index(batchData))
# save hdf5 file
# second postprocessing find extrema if needed also for variables defined over space
## TODO: fix data saving methods as it will not work now
for locationOfInterest in self.locationsOfInterest.values():
locationOfInterest.preprocessSolutionDataExtremaAndInflectionPoints(
self.simulationTime, self.sampleSize)
#for locationOfInterest in self.locationsOfInterest.values():
# locationOfInterest.preprocessSolutionDataTrajectory(self.simulationTime, self.sampleSize)
self.closeAndSaveQuantityOfInterestFile()
def MacCormack_Field(self):
"""
MacCormack solver method with forward-euler time steping,
Using either Characteristic system 0 or 1 as defined in the XML-file.
This method is solving the system by looping through the defined network
imposing the boundary conditions based on Riemann Invariants and then solving the vessels,
conncetions, bifucations with a predictor-corrector step method
"""
if self.quiet == False:
logger.info("Solving system ...")
progressBar = cPB.ProgressBar(35,self.nTSteps, subpressPrint = self.quiet)
reflectionCoefficientCount = 0
maxRef = 0
if self.cycleMode == False:
# original
for n in xrange(self.nTSteps):
self.currentTimeStep[0] = n
self.currentMemoryIndex[0] = n - self.memoryOffset[0]
if self.quiet == False:
progressBar.progress(n)
for numericalObject in self.numericalObjects:
try:
numericalObject()
except Exception:
# Save the Solution data for debugging
logger.critical("Exception caught in {} by MacCormack_Field attempting to save solution data file...".format(numericalObject))
self.vascularNetwork.runtimeMemoryManager.flushSolutionMemory()
self.vascularNetwork.saveSolutionData()
logger.critical("Success in saving solution data file. Reraising Exception")
raise # TODO: why does self.exception() not force the program to quit?
# self.exception()
if self.quiet == False:
progressBar.progress(n)
## to be concentrated with original cycle mode !!
else:
# steady state variables
P_lastCycle = {}
Q_lastCycle = {}
A_lastCycle = {}
for vesselId,vessel in self.vessels.iteritems():
initialValues = self.vascularNetwork.initialValues
p0,p1 = initialValues[vesselId]['Pressure']
Qm = initialValues[vesselId]['Flow']
P_lastCycle[vesselId] = np.ones((self.nTSteps,vessel.N))
Q_lastCycle[vesselId] = np.ones((self.nTSteps,vessel.N))
A_lastCycle[vesselId] = np.ones((self.nTSteps,vessel.N))
for cycle in xrange(self.numberCycles-1):
logger.info(' solving cycle {}'.format(cycle+1))
# 1. solve cycle
for n in xrange(self.nTSteps-1):
self.currentTimeStep[0] = n
for numericalObject in self.numericalObjects:
numericalObject()
# 2. check for steady state
if self.quiet == False:
for vesselId in self.vessels.keys():
#Perror = np.sum(np.sqrt((np.divide((P_lastCycle[vesselId]-self.P[vesselId]),P_lastCycle[vesselId]))**2.0))/self.P[vesselId].size
#Qerror = np.sum(np.sqrt((np.divide((Q_lastCycle[vesselId]-self.Q[vesselId]),Q_lastCycle[vesselId]))**2.0))/self.Q[vesselId].size
#Aerror = np.sum(np.sqrt((np.divide((A_lastCycle[vesselId]-self.A[vesselId]),A_lastCycle[vesselId]))**2.0))/self.A[vesselId].size
Perror = np.max(np.sqrt((np.divide((P_lastCycle[vesselId]-self.P[vesselId]),P_lastCycle[vesselId]))**2.0))
#Qerror = np.max(np.sqrt((np.divide((Q_lastCycle[vesselId]-self.Q[vesselId]),Q_lastCycle[vesselId]))**2.0))
Aerror = np.max(np.sqrt((np.divide((A_lastCycle[vesselId]-self.A[vesselId]),A_lastCycle[vesselId]))**2.0))
P_lastCycle[vesselId] = self.P[vesselId].copy()
#Q_lastCycle[vesselId] = self.Q[vesselId].copy()
A_lastCycle[vesselId] = self.A[vesselId].copy()
logger.debug('{} {}'.format(Perror, Aerror))
# 3. rehash solution arrays if not last cycle
if cycle is not self.numberCycles-1:
for vesselId in self.vessels.keys():
self.P[vesselId][0] = self.P[vesselId][-1]
self.Q[vesselId][0] = self.Q[vesselId][-1]
self.A[vesselId][0] = self.A[vesselId][-1]
if self.quiet == False: logger.info("\nSystem solved!")
BVcheck = False
if BVcheck == True:
logger.info('\n=====================================')
logger.info('___Blood volume - consistency check___')
logger.info('Vessels init (ml) sol (ml) diff')
vesselsInit = 0
vesselsSol = 0
for vesselId,vessel in self.vessels.iteritems():
A1 = self.vessels[vesselId].Asol[0][0:-1]
A2 = self.vessels[vesselId].Asol[0][1:]
volumeInit = np.sum(vessel.dz*(A1+A2+np.sqrt(A1*A2))/3.0)*1.e6
A1 = self.vessels[vesselId].Asol[-1][0:-1]
A2 = self.vessels[vesselId].Asol[-1][1:]
volumeSol = np.sum(vessel.dz*(A1+A2+np.sqrt(A1*A2))/3.0)*1.e6
diff = volumeInit-volumeSol
vesselsInit += volumeInit
vesselsSol += volumeSol
logger.info('{:<4} {:7.2f} {:7.2f} {:4.2f}'.format(str(vesselId), volumeInit, volumeSol, diff))
logger.info(' ----------------------------------- ')
logger.info('Boundarys in (ml) out (ml) ')
totalIn = 0
totalOut = 0
for boundaryList in self.boundarys.itervalues():
for boundary in boundaryList:
logger.info('{:<12} {:6.2f} {:4.2f}'.format(boundary.name,
abs(boundary.BloodVolumen[0])*1.e6,
abs(boundary.BloodVolumen[1])*1.e6 ))
totalIn += abs(boundary.BloodVolumen[0])*1.e6
totalOut += abs(boundary.BloodVolumen[1])*1.e6
logger.info(' ----------------------------------- ')
logger.info('total')
logger.info(' vessels initial (ml) {:4.2f}'.format(vesselsInit))
logger.info(' vessels solution (ml) {:4.2f}'.format(vesselsSol))
logger.info(' boundarys in (ml) {:4.2f}'.format(totalIn))
logger.info(' boundarys out (ml) {:4.2f}'.format(totalOut))
logger.info(' ------------')
logger.info(' volume diff (ml) {:.2f}'.format(vesselsInit - vesselsSol + totalIn - totalOut))
## stop realtime visualisation
for communicator in self.communicators.itervalues():
try: communicator.stopRealtimeViz()
except Exception: self.warning("old except: pass #1 clause in c1dFlowSolv.MacCormack_Field", oldExceptPass= True)
### garbage collection
gc.collect()
del self.numericalObjects
del self.fields
del self.connections
del self.boundarys
del self.communicators