def runBatchAsSingleProcess(batchDataList, quiet = False): ''' Run a set of simulations on one core without multiprocessing Args: batchDataList (list) : with data for each batch job [batchData1, batchData .. ] batchData (dict) : dict with {simulationIndex: , networkName: , dataNumber: , networkXmlFile: , pathSolutionDataFilename: } ''' timeStartBatch = time.time() print '=====================================' print '------Single Process Batch Job-------' print 'numberOfEval.: {}'.format(len(batchDataList)) progressBar = cPB.ProgressBar(35, len(batchDataList)) for completed,batchData in enumerate(batchDataList): minutesSolve,secsSolve = runSingleBatchSimulation(batchData) if quiet == False: print '____________Batch {:5} ___________'.format(batchDataList.index(batchData)) print 'Runtime: {} min {} sec'.format(minutesSolve,secsSolve) progressBar.progress(completed) timeBatchJob= time.time()-timeStartBatch minutesBatch = int(timeBatchJob/60.) secsBatch = timeBatchJob-minutesBatch*60. print '=====================================' print 'total runtime: {} min {} sec'.format(minutesBatch,secsBatch) print '====================================='
def createEvaluationCaseFiles(self): ''' batchDataList <list> := with data for each batch job [batchData1, batchData .. ] batchData <dict> := dict with {simulationIndex: , networkName: , dataNumber: , networkXmlFile: , pathSolutionDataFilename: } ''' self.evaluationCaseFiles = [ ] # list of dict: [ caseFileDict1,caseFileDict2 ..] for each evaluation #TODO: replace create evaluation files or save them to disc!!! if self.simulateEvaluations == True or self.preProcessData == True or self.createEvaluationXmlFiles == True: sampleSize = self.sampleManager.currentSampleSize print "Create evaluation case file list" progressBar = cPB.ProgressBar(35, sampleSize) for simulationIndex in xrange(sampleSize): networkXmlFileLoad = mFPH_VPC.getFilePath( 'uqsaEvaluationNetworkXmlFile', self.networkName, self.dataNumber, 'write', caseName=self.sampleManager.samplingMethod, evaluationNumber=simulationIndex) networkXmlFileSave = networkXmlFileLoad pathSolutionDataFilename = mFPH_VPC.getFilePath( 'uqsaEvaluationSolutionDataFile', self.networkName, self.dataNumber, 'write', caseName=self.sampleManager.samplingMethod, evaluationNumber=simulationIndex) caseFileDict1 = { 'simulationIndex': simulationIndex, 'networkName': self.networkName, 'dataNumber': self.dataNumber, 'networkXmlFileLoad': networkXmlFileLoad, 'networkXmlFileSave': networkXmlFileSave, 'pathSolutionDataFilename': pathSolutionDataFilename } self.evaluationCaseFiles.append(caseFileDict1) progressBar.progress(simulationIndex)
def hashDataForGivenBases(self, basis, sampleSize): ''' ''' self.hdf5Group.create_dataset('trajectoryData', (sampleSize,len(basis)) , dtype='float64') self.hdf5Group.create_dataset('trajectoryBasis', data = basis, dtype='float64') progressBar = cPB.ProgressBar(35, sampleSize) for n in xrange(sampleSize): data = self.hdf5Group['data'][n] dataBasis = self.hdf5Group['dataBasis'][n] self.hdf5Group['trajectoryData'][n] = np.interp(basis, dataBasis, data) progressBar.progress(n)
def runBatchAsMultiprocessing(batchDataList, numberWorkers = None, quiet = False): ''' Run a set of simulations on one core without multiprocessing Args: batchDataList (list) : with data for each batch job [batchData1, batchData .. ] batchData (dict) : dict with {simulationIndex: , networkName: , dataNumber: , networkXmlFile: , pathSolutionDataFilename: } ''' if numberWorkers == None: numberWorkers = multiprocessing.cpu_count() timeStartBatch = time.time() print '=====================================' print '------Multiprocessing Batch Job------' print 'numberWorkers: {}'.format(numberWorkers) print 'numberOfEval.: {}'.format(len(batchDataList)) progressBar = cPB.ProgressBar(35, len(batchDataList)) pool = multiprocessing.Pool(numberWorkers, maxtasksperchild = None) results = pool.imap(runSingleBatchSimulation,batchDataList) pool.close() while (True): completed = results._index if (completed == len(batchDataList)): break progressBar.progress(completed) pool.join() if quiet == False: print '=====================================' for batchJobIndex,[minutesSolve,secsSolve] in enumerate(results): print '____________Batch {:5} ___________'.format(batchJobIndex+1) print 'Runtime: {} min {} sec'.format(minutesSolve,secsSolve) print '=====================================' timeBatchJob= time.time()-timeStartBatch minutesBatch = int(timeBatchJob/60.) secsBatch = timeBatchJob-minutesBatch*60. print 'total runtime: {} min {} sec'.format(minutesBatch,secsBatch) print '====================================='
def preprocessSolutionData(self, evaluationCaseFiles, preprocessedSolutionData, simulationTimeFileSave, simulationTimeFileLoad): ''' load all samples and pass data to locations of interest find simulation time array with largest dt invoke time cropping and post processing for the data ''' # loop through all solution data files # find time array #solutionTime min nPoints = 1e32 timeS = 0 timeE = 1e32 if self.evaluateSimulationTime == True: print "estimate simulation time of all simulations" progressBar = cPB.ProgressBar(35, len(evaluationCaseFiles)) for batchData in evaluationCaseFiles: networkName = batchData['networkName'] dataNumber = batchData['dataNumber'] networkXmlFileLoad = batchData['networkXmlFileLoad'] pathSolutionDataFilename = batchData[ 'pathSolutionDataFilename'] vascularNetwork = moduleXML.loadNetworkFromXML( networkName, dataNumber, networkXmlFile=networkXmlFileLoad, pathSolutionDataFilename=pathSolutionDataFilename) vascularNetwork.linkSolutionData() cPoints = len(vascularNetwork.simulationTime) ctimeS = min(vascularNetwork.simulationTime) ctimeE = max(vascularNetwork.simulationTime) if cPoints < nPoints: nPoints = cPoints if ctimeS > timeS: timeS = ctimeS if ctimeE < timeE: timeE = ctimeE vascularNetwork.solutionDataFile.close() del vascularNetwork progressBar.progress(evaluationCaseFiles.index(batchData)) self.simulationTime = np.linspace(timeS, timeE, nPoints) # save the simulationTime self.saveQuantitiyOfInterestData(simulationTimeFileSave) else: print "load estimated simulation time of all simulations" if simulationTimeFileLoad != None: self.loadQuantitiyOfInterestData(simulationTimeFileLoad) else: raise ValueError( 'simulationTime hdf5 file for case does not exist! {}'. format(simulationTimeFileLoad)) self.openQuantityOfInterestFile(preprocessedSolutionData) print "estimate solution data for quantities of interest" progressBar = cPB.ProgressBar(35, len(evaluationCaseFiles)) # pass the data to the locationsOfInterests which will load the information needed for batchData in evaluationCaseFiles: simulationIndex = batchData['simulationIndex'] networkName = batchData['networkName'] dataNumber = batchData['dataNumber'] networkXmlFileLoad = batchData['networkXmlFileLoad'] pathSolutionDataFilename = batchData['pathSolutionDataFilename'] vascularNetwork = moduleXML.loadNetworkFromXML( networkName, dataNumber, networkXmlFile=networkXmlFileLoad, pathSolutionDataFilename=pathSolutionDataFilename) vascularNetwork.linkSolutionData() for locationOfInterest in self.locationsOfInterest.values(): locationOfInterest.preprocessSolutionData( vascularNetwork, self.simulationTime, self.sampleSize, simulationIndex) progressBar.progress(evaluationCaseFiles.index(batchData)) # save hdf5 file # second postprocessing find extrema if needed also for variables defined over space ## TODO: fix data saving methods as it will not work now for locationOfInterest in self.locationsOfInterest.values(): locationOfInterest.preprocessSolutionDataExtremaAndInflectionPoints( self.simulationTime, self.sampleSize) #for locationOfInterest in self.locationsOfInterest.values(): # locationOfInterest.preprocessSolutionDataTrajectory(self.simulationTime, self.sampleSize) self.closeAndSaveQuantityOfInterestFile()
def MacCormack_Field(self): """ MacCormack solver method with forward-euler time steping, Using either Characteristic system 0 or 1 as defined in the XML-file. This method is solving the system by looping through the defined network imposing the boundary conditions based on Riemann Invariants and then solving the vessels, conncetions, bifucations with a predictor-corrector step method """ if self.quiet == False: logger.info("Solving system ...") progressBar = cPB.ProgressBar(35,self.nTSteps, subpressPrint = self.quiet) reflectionCoefficientCount = 0 maxRef = 0 if self.cycleMode == False: # original for n in xrange(self.nTSteps): self.currentTimeStep[0] = n self.currentMemoryIndex[0] = n - self.memoryOffset[0] if self.quiet == False: progressBar.progress(n) for numericalObject in self.numericalObjects: try: numericalObject() except Exception: # Save the Solution data for debugging logger.critical("Exception caught in {} by MacCormack_Field attempting to save solution data file...".format(numericalObject)) self.vascularNetwork.runtimeMemoryManager.flushSolutionMemory() self.vascularNetwork.saveSolutionData() logger.critical("Success in saving solution data file. Reraising Exception") raise # TODO: why does self.exception() not force the program to quit? # self.exception() if self.quiet == False: progressBar.progress(n) ## to be concentrated with original cycle mode !! else: # steady state variables P_lastCycle = {} Q_lastCycle = {} A_lastCycle = {} for vesselId,vessel in self.vessels.iteritems(): initialValues = self.vascularNetwork.initialValues p0,p1 = initialValues[vesselId]['Pressure'] Qm = initialValues[vesselId]['Flow'] P_lastCycle[vesselId] = np.ones((self.nTSteps,vessel.N)) Q_lastCycle[vesselId] = np.ones((self.nTSteps,vessel.N)) A_lastCycle[vesselId] = np.ones((self.nTSteps,vessel.N)) for cycle in xrange(self.numberCycles-1): logger.info(' solving cycle {}'.format(cycle+1)) # 1. solve cycle for n in xrange(self.nTSteps-1): self.currentTimeStep[0] = n for numericalObject in self.numericalObjects: numericalObject() # 2. check for steady state if self.quiet == False: for vesselId in self.vessels.keys(): #Perror = np.sum(np.sqrt((np.divide((P_lastCycle[vesselId]-self.P[vesselId]),P_lastCycle[vesselId]))**2.0))/self.P[vesselId].size #Qerror = np.sum(np.sqrt((np.divide((Q_lastCycle[vesselId]-self.Q[vesselId]),Q_lastCycle[vesselId]))**2.0))/self.Q[vesselId].size #Aerror = np.sum(np.sqrt((np.divide((A_lastCycle[vesselId]-self.A[vesselId]),A_lastCycle[vesselId]))**2.0))/self.A[vesselId].size Perror = np.max(np.sqrt((np.divide((P_lastCycle[vesselId]-self.P[vesselId]),P_lastCycle[vesselId]))**2.0)) #Qerror = np.max(np.sqrt((np.divide((Q_lastCycle[vesselId]-self.Q[vesselId]),Q_lastCycle[vesselId]))**2.0)) Aerror = np.max(np.sqrt((np.divide((A_lastCycle[vesselId]-self.A[vesselId]),A_lastCycle[vesselId]))**2.0)) P_lastCycle[vesselId] = self.P[vesselId].copy() #Q_lastCycle[vesselId] = self.Q[vesselId].copy() A_lastCycle[vesselId] = self.A[vesselId].copy() logger.debug('{} {}'.format(Perror, Aerror)) # 3. rehash solution arrays if not last cycle if cycle is not self.numberCycles-1: for vesselId in self.vessels.keys(): self.P[vesselId][0] = self.P[vesselId][-1] self.Q[vesselId][0] = self.Q[vesselId][-1] self.A[vesselId][0] = self.A[vesselId][-1] if self.quiet == False: logger.info("\nSystem solved!") BVcheck = False if BVcheck == True: logger.info('\n=====================================') logger.info('___Blood volume - consistency check___') logger.info('Vessels init (ml) sol (ml) diff') vesselsInit = 0 vesselsSol = 0 for vesselId,vessel in self.vessels.iteritems(): A1 = self.vessels[vesselId].Asol[0][0:-1] A2 = self.vessels[vesselId].Asol[0][1:] volumeInit = np.sum(vessel.dz*(A1+A2+np.sqrt(A1*A2))/3.0)*1.e6 A1 = self.vessels[vesselId].Asol[-1][0:-1] A2 = self.vessels[vesselId].Asol[-1][1:] volumeSol = np.sum(vessel.dz*(A1+A2+np.sqrt(A1*A2))/3.0)*1.e6 diff = volumeInit-volumeSol vesselsInit += volumeInit vesselsSol += volumeSol logger.info('{:<4} {:7.2f} {:7.2f} {:4.2f}'.format(str(vesselId), volumeInit, volumeSol, diff)) logger.info(' ----------------------------------- ') logger.info('Boundarys in (ml) out (ml) ') totalIn = 0 totalOut = 0 for boundaryList in self.boundarys.itervalues(): for boundary in boundaryList: logger.info('{:<12} {:6.2f} {:4.2f}'.format(boundary.name, abs(boundary.BloodVolumen[0])*1.e6, abs(boundary.BloodVolumen[1])*1.e6 )) totalIn += abs(boundary.BloodVolumen[0])*1.e6 totalOut += abs(boundary.BloodVolumen[1])*1.e6 logger.info(' ----------------------------------- ') logger.info('total') logger.info(' vessels initial (ml) {:4.2f}'.format(vesselsInit)) logger.info(' vessels solution (ml) {:4.2f}'.format(vesselsSol)) logger.info(' boundarys in (ml) {:4.2f}'.format(totalIn)) logger.info(' boundarys out (ml) {:4.2f}'.format(totalOut)) logger.info(' ------------') logger.info(' volume diff (ml) {:.2f}'.format(vesselsInit - vesselsSol + totalIn - totalOut)) ## stop realtime visualisation for communicator in self.communicators.itervalues(): try: communicator.stopRealtimeViz() except Exception: self.warning("old except: pass #1 clause in c1dFlowSolv.MacCormack_Field", oldExceptPass= True) ### garbage collection gc.collect() del self.numericalObjects del self.fields del self.connections del self.boundarys del self.communicators