def run(self): if not self.opts.keepPostprocessing: self.opts.additional.append("postProcessing") notCleared=[] if self.opts.dryRun: self.opts.verbose=True self.opts.verboseClear=True for cName in self.parser.getArgs(): if self.checkCase(cName,fatal=self.opts.fatal,verbose=not self.opts.silent): if self.opts.allcleanScript: scr=path.join(cName,"Allclean") if(path.exists(scr)): if self.opts.verbose or self.opts.verboseClear: print("Executing",scr) execute(scr,workdir=cName) try: self.addLocalConfig(cName) if self.opts.verbose: print_("Clearing",cName) sol=SolutionDirectory(cName, archive=None, parallel=True, paraviewLink=False) sol.clear(after=self.parser.getOptions().after, processor=self.parser.getOptions().processor, pyfoam=self.parser.getOptions().pyfoam, vtk=self.parser.getOptions().vtk, verbose=self.parser.getOptions().verboseClear, removeAnalyzed=self.parser.getOptions().removeAnalyzed, keepRegular=self.parser.getOptions().keepRegular, keepParallel=self.parser.getOptions().keepParallel, keepLast=self.parser.getOptions().latest, keepInterval=self.parser.getOptions().keepInterval, keepTimes=self.opts.keepTimes, clearHistory=self.parser.getOptions().clearHistory, clearParameters=self.parser.getOptions().clearParameters, additional=self.parser.getOptions().additional, dryRun=self.opts.dryRun, functionObjectData=self.parser.getOptions().functionObjectData) self.addToCaseLog(cName) except OSError: e = sys.exc_info()[1] # compatible with 2.x and 3.x self.warning("Can not clear",cName,"because of OSError",e) notCleared.append(cName) if len(notCleared)>0: self.warning("These case not cleared because of OS-problems:", ", ".join(notCleared))
def execute(self,para,log): cmd=replaceValues(self.command,para) print_(" Executing ",cmd,end=" ") sys.stdout.flush() out=execute(cmd) if len(out)>0: print_(" -->",len(out),"lines output") for l in out: print_("---- Command:",cmd,file=log) print_(l,end=" ",file=log) else: print_() return True,None
def executeScript(self, scriptName, workdir, echo): """Execute a script and write a corresponding logfile""" ret, txt = execute([scriptName], workdir=workdir, echo=echo, getReturnCode=True) result = "".join(txt) open(scriptName + ".log", "w").write(result) if ret not in [0, None]: self.info(scriptName, "failed with code", ret) if self.opts.failOnScriptFailure: self.error("Script", scriptName, "failed with code", ret)
def execute(self,para,log): cmd=replaceValues(self.command,para) print " Executing ",cmd, sys.stdout.flush() out=execute(cmd) if len(out)>0: print " -->",len(out),"lines output" for l in out: print >>log,"---- Command:",cmd print >>log,l, else: print return True,None
def run(self): config=ConfigParser.ConfigParser() files=self.parser.getArgs() good=config.read(files) # will work with 2.4 # if len(good)!=len(files): # print "Problem while trying to parse files",files # print "Only ",good," could be parsed" # sys.exit(-1) benchName=config.get("General","name") if self.opts.nameAddition!=None: benchName+="_"+self.opts.nameAddition if self.opts.foamVersion!=None: benchName+="_v"+self.opts.foamVersion isParallel=config.getboolean("General","parallel") lam=None if isParallel: nrCpus=config.getint("General","nProcs") machineFile=config.get("General","machines") if not path.exists(machineFile): self.error("Machine file ",machineFile,"needed for parallel run") lam=LAMMachine(machineFile,nr=nrCpus) if lam.cpuNr()>nrCpus: self.error("Wrong number of CPUs: ",lam.cpuNr()) print "Running parallel on",lam.cpuNr(),"CPUs" if config.has_option("General","casesDirectory"): casesDirectory=path.expanduser(config.get("General","casesDirectory")) else: casesDirectory=foamTutorials() if not path.exists(casesDirectory): self.error("Directory",casesDirectory,"needed with the benchmark cases is missing") else: print "Using cases from directory",casesDirectory benchCases=[] config.remove_section("General") for sec in config.sections(): print "Reading: ",sec skipIt=False skipReason="" if config.has_option(sec,"skip"): skipIt=config.getboolean(sec,"skip") skipReason="Switched off in file" if self.opts.excases!=None and not skipIt: for p in self.opts.excases: if fnmatch(sec,p): skipIt=True skipReason="Switched off by pattern '"+p+"'" if self.opts.cases!=None: for p in self.opts.cases: if fnmatch(sec,p): skipIt=False skipReason="" if skipIt: print "Skipping case ..... Reason:"+skipReason continue sol=config.get(sec,"solver") cas=config.get(sec,"case") pre=eval(config.get(sec,"prepare")) preCon=[] if config.has_option(sec,"preControlDict"): preCon=eval(config.get(sec,"preControlDict")) con=eval(config.get(sec,"controlDict")) bas=config.getfloat(sec,"baseline") wei=config.getfloat(sec,"weight") add=[] if config.has_option(sec,"additional"): add=eval(config.get(sec,"additional")) print "Adding: ", add util=[] if config.has_option(sec,"utilities"): util=eval(config.get(sec,"utilities")) print "Utilities: ", util nr=99999 if config.has_option(sec,"nr"): nr=eval(config.get(sec,"nr")) sp=None if config.has_option(sec,"blockSplit"): sp=eval(config.get(sec,"blockSplit")) toRm=[] if config.has_option(sec,"filesToRemove"): toRm=eval(config.get(sec,"filesToRemove")) setInit=[] if config.has_option(sec,"setInitial"): setInit=eval(config.get(sec,"setInitial")) parallelOK=False if config.has_option(sec,"parallelOK"): parallelOK=config.getboolean(sec,"parallelOK") deMet=["metis"] if config.has_option(sec,"decomposition"): deMet=config.get(sec,"decomposition").split() if deMet[0]=="metis": pass elif deMet[0]=="simple": if len(deMet)<2: deMet.append(0) else: deMet[1]=int(deMet[1]) else: print "Unimplemented decomposition method",deMet[0],"switching to metis" deMet=["metis"] if isParallel==False or parallelOK==True: if path.exists(path.join(casesDirectory,sol,cas)): benchCases.append( (nr,sec,sol,cas,pre,con,preCon,bas,wei,add,util,sp,toRm,setInit,deMet) ) else: print "Skipping",sec,"because directory",path.join(casesDirectory,sol,cas),"could not be found" else: print "Skipping",sec,"because not parallel" benchCases.sort() parallelString="" if isParallel: parallelString=".cpus="+str(nrCpus) resultFile=open("Benchmark."+benchName+"."+uname()[1]+parallelString+".results","w") totalSpeedup=0 minSpeedup=None maxSpeedup=None totalWeight =0 runsOK=0 currentEstimate = 1. print "\nStart Benching\n" csv=CSVCollection("Benchmark."+benchName+"."+uname()[1]+parallelString+".csv") # csvHeaders=["description","solver","case","caseDir","base", # "benchmark","machine","arch","cpus","os","version", # "wallclocktime","cputime","cputimeuser","cputimesystem","maxmemory","cpuusage","speedup"] for nr,description,solver,case,prepare,control,preControl,base,weight,additional,utilities,split,toRemove,setInit,decomposition in benchCases: # control.append( ("endTime",-2000) ) print "Running Benchmark: ",description print "Solver: ",solver print "Case: ",case caseName=solver+"_"+case+"_"+benchName+"."+uname()[1]+".case" print "Short name: ",caseName caseDir=caseName+".runDir" csv["description"]=description csv["solver"]=solver csv["case"]=case csv["caseDir"]=caseDir csv["base"]=base csv["benchmark"]=benchName csv["machine"]=uname()[1] csv["arch"]=uname()[4] if lam==None: csv["cpus"]=1 else: csv["cpus"]=lam.cpuNr() csv["os"]=uname()[0] csv["version"]=uname()[2] workDir=path.realpath(path.curdir) orig=SolutionDirectory(path.join(casesDirectory,solver,case), archive=None, paraviewLink=False) for a in additional+utilities: orig.addToClone(a) orig.cloneCase(path.join(workDir,caseDir)) if oldApp(): argv=[solver,workDir,caseDir] else: argv=[solver,"-case",path.join(workDir,caseDir)] run=BasicRunner(silent=True,argv=argv,logname="BenchRunning",lam=lam) runDir=run.getSolutionDirectory() controlFile=ParameterFile(runDir.controlDict()) for name,value in preControl: print "Setting parameter",name,"to",value,"in controlDict" controlFile.replaceParameter(name,value) for rm in toRemove: fn=path.join(caseDir,rm) print "Removing file",fn remove(fn) for field,bc,val in setInit: print "Setting",field,"on",bc,"to",val SolutionFile(runDir.initialDir(),field).replaceBoundary(bc,val) oldDeltaT=controlFile.replaceParameter("deltaT",0) for u in utilities: print "Building utility ",u execute("wmake 2>&1 >%s %s" % (path.join(caseDir,"BenchCompile."+u),path.join(caseDir,u))) print "Preparing the case: " if lam!=None: prepare=prepare+[("decomposePar","")] if decomposition[0]=="metis": lam.writeMetis(SolutionDirectory(path.join(workDir,caseDir))) elif decomposition[0]=="simple": lam.writeSimple(SolutionDirectory(path.join(workDir,caseDir)),decomposition[1]) if split: print "Splitting the mesh:",split bm=BlockMesh(runDir.blockMesh()) bm.refineMesh(split) for pre,post in prepare: print "Doing ",pre," ...." post=post.replace("%case%",caseDir) if oldApp(): args=string.split("%s %s %s %s" % (pre,workDir,caseDir,post)) else: args=string.split("%s -case %s %s" % (pre,path.join(workDir,caseDir),post)) util=BasicRunner(silent=True,argv=args,logname="BenchPrepare_"+pre) util.start() controlFile.replaceParameter("deltaT",oldDeltaT) # control.append(("endTime",-1000)) for name,value in control: print "Setting parameter",name,"to",value,"in controlDict" controlFile.replaceParameter(name,value) print "Starting at ",asctime(localtime(time())) print " Baseline is %f, estimated speedup %f -> estimated end at %s " % (base,currentEstimate,asctime(localtime(time()+base/currentEstimate))) print "Running the case ...." run.start() speedup=None cpuUsage=0 speedupOut=-1 try: speedup=base/run.run.wallTime() cpuUsage=100.*run.run.cpuTime()/run.run.wallTime() except ZeroDivisionError: print "Division by Zero: ",run.run.wallTime() if not run.runOK(): print "\nWARNING!!!!" print "Run had a problem, not using the results. Check the log\n" speedup=None if speedup!=None: speedupOut=speedup totalSpeedup+=speedup*weight totalWeight +=weight runsOK+=1 if maxSpeedup==None: maxSpeedup=speedup elif speedup>maxSpeedup: maxSpeedup=speedup if minSpeedup==None: minSpeedup=speedup elif speedup<minSpeedup: minSpeedup=speedup print "Wall clock: ",run.run.wallTime() print "Speedup: ",speedup," (Baseline: ",base,")" print "CPU Time: ",run.run.cpuTime() print "CPU Time User: "******"CPU Time System: ",run.run.cpuSystemTime() print "Memory: ",run.run.usedMemory() print "CPU Usage: %6.2f%%" % (cpuUsage) csv["wallclocktime"]=run.run.wallTime() csv["cputime"]=run.run.cpuTime() csv["cputimeuser"]=run.run.cpuUserTime() csv["cputimesystem"]=run.run.cpuSystemTime() csv["maxmemory"]=run.run.usedMemory() csv["cpuusage"]=cpuUsage if speedup!=None: csv["speedup"]=speedup else: csv["speedup"]="##" csv.write() resultFile.write("Case %s WallTime %g CPUTime %g UserTime %g SystemTime %g Memory %g MB Speedup %g\n" %(caseName,run.run.wallTime(),run.run.cpuTime(),run.run.cpuUserTime(),run.run.cpuSystemTime(),run.run.usedMemory(),speedupOut)) resultFile.flush() if speedup!=None: currentEstimate=totalSpeedup/totalWeight if self.opts.removeCases: print "Clearing case", if speedup==None: print "not ... because it failed" else: print "completely" rmtree(caseDir,ignore_errors=True) print print if lam!=None: lam.stop() print "Total Speedup: ",currentEstimate," ( ",totalSpeedup," / ",totalWeight, " ) Range: [",minSpeedup,",",maxSpeedup,"]" print runsOK,"of",len(benchCases),"ran OK" resultFile.write("Total Speedup: %g\n" % (currentEstimate)) if minSpeedup and maxSpeedup: resultFile.write("Range: [ %g , %g ]\n" % (minSpeedup,maxSpeedup)) resultFile.close()
def run(self): cfgFile=self.parser.getArgs()[0] moduleFile=self.parser.getArgs()[1] useCsh = self.opts.useCshell if self.opts.clearEnvironment: self.warning("Clearing environment of variables that might come from an OpenFAOM-installation. Nevertheless it is preferable to use a clean environment") try: oldVersion=os.environ["WM_PROJECT_VERSION"] except KeyError: self.warning("Seems to be a clean environment anyway") oldVersion=None if oldVersion: for e in os.environ.keys(): for p in ["WM_","FOAM_"]: if e.find(p)==0: del os.environ[e] break for p in ["PATH","DYLD_LIBRARY_PATH","LD_LIBRARY_PATH","MANPATH","PV_PLUGIN_PATH"]: if p in os.environ: lst=os.environ[p].split(":") os.environ[p]=":".join([l for l in lst if l.find(oldVersion)<0]) if path.exists(cfgFile) and path.isfile(cfgFile) and os.access(cfgFile, os.R_OK): print " Using file " + cfgFile + " for loading the OpenFOAM environment" # Some more sanity check fileData = open(cfgFile).read() if not useCsh and "setenv" in fileData: self.error(" Error: Detecting 'setenv' instructions in this bash file. Please provide a configuration file compatible for bash, or use the --csh option.") elif useCsh and "export" in fileData: self.error(" Error: Detecting 'export' instructions in this csh file. Please provide a configuration file compatible for csh.") else: print " The configuration file seems ok" else: self.error(" Error: Cannot access file: " + cfgFile) # We choose not to run if the OpenFOAM environment is already loaded. # We obviously cannot diff from such an environment if os.getenv("WM_PROJECT") != None: self.error(""" Error: Cannot run with OpenFOAM environment variables already present You need to run this script from a clean environment""") # We choose not to run under the user 'root', simply because the HOME and USER environment # variables are usually very similar for this user, and this will cause problem later on # For instance, under Centos, for the super user: # HOME=/root # USER=root # Those two are very similar if os.getenv("USER") == "root": self.error(""" Error: You should not run this script from the 'root' account. Please invoke this pyFoam command from a plain OpenFOAM user account. Then, make the result file available to the super user (root) of the" destination host in order to install the modules configuration files.""") # Grab environment + aliases before the activation of OpenFOAM # We start from a minimal environment using 'env -i' shellCmd = 'bash -c' sourceCmd = '.' if useCsh: shellCmd = 'csh -c' sourceCmd = 'source' oldEnv=Set(execute(shellCmd + ' "env|sort"')) oldAliases=Set(execute(shellCmd + ' "alias|sort"')) # Grab environment + aliases after the activation of OpenFOAM newEnv=Set(execute(shellCmd + ' \"' + sourceCmd + ' ' + cfgFile + '; env|sort"')) newAliases=Set(execute(shellCmd + ' \"' + sourceCmd + ' ' + cfgFile + '; alias|sort"')) # Handling of environment variables # Memorize all the PATH environment variables for later processing oldPath = {} newPath = {} homeVar = "" userVar = "" for v in oldEnv: if v.find('=') > -1: key, value = v.split('=', 1) # Memorize PATH variables if key.find('PATH') > -1: oldPath[key] = value.replace('\n', '') # We remove any reference to PyFOAM in the old PATH env. variable # Since we run this script with PyFOAM, environment variables for PyFoam are # present in oldPath. If the configuration file for OpenFOAM requires PyFoam too, # we risk to lose these variables because they will be in common in both the # 'before' and 'after'environment. # This is ugly, but PyFOAM is just too useful to write this script with plain # python. # If you have installed PyFoam under a path where the string 'PyFoam' is not present, # you will simply have to manually add the necessary PyFoam environment variables # yourself in the module file. No biggie, oldPath[key] = self.removeComponentWithSubstrFromPathEnvVar(oldPath[key], 'PyFoam') # Memorize HOME variable if key == 'HOME': homeVar = value.replace('\n', '') # Memorize USER variable if key == 'USER': userVar = value.replace('\n', '') # Using sets, it is trivial to remove common environment variables # and aliases between the two environments moduleEnv=newEnv - oldEnv moduleAliases=newAliases - oldAliases # Dictionary for environment variables envVar = {} # Dictionary for aliases aliasesVar = {} # Iterate through environment variables and store in dictionary for v in moduleEnv: if v.find('=') > -1: key, value = v.split('=') # Minor cleanup value = value.replace('\n', '') # Memorize PATH variables for later processing if key.find('PATH') > -1: newPath[key] = value else: envVar.setdefault(key, []).append(value) # Handle the PATH variables for v in newPath: if v in oldPath: # Cleanup old PATH components newPath[v] = self.removeComponentFromPathEnvVar(newPath[v], oldPath[v]) # Remove duplicate entries in PATH variable newPath[v] = self.uniqifyPathEnvVar(newPath[v]) envVar.setdefault(v, []).append(newPath[v]) # Iterate through aliases variables and store them in dictionary for v in moduleAliases: if v.find('=') > -1: key, value = v.split('=', 1) # Minor cleanup key = key.replace("alias ", "") value = value.replace('\n', "") value = value.replace('\'', "") aliasesVar.setdefault(key, []).append(value) # Generate module entries print " Generating modulefile: " , moduleFile f = open(moduleFile, 'w') self.writeModuleFileHeader(f, envVar["WM_PROJECT_VERSION"][0]) self.writeModuleEnvironmentEntry(f, envVar, homeVar, userVar) self.writeModuleAliasEntry(f, aliasesVar, homeVar, userVar) self.writeModuleFileFooter(f) f.close() print " Done\n"
def run(self): cfgFile = self.parser.getArgs()[0] moduleFile = self.parser.getArgs()[1] useCsh = self.opts.useCshell if self.opts.clearEnvironment: self.warning( "Clearing environment of variables that might come from an OpenFAOM-installation. Nevertheless it is preferable to use a clean environment" ) try: oldVersion = os.environ["WM_PROJECT_VERSION"] except KeyError: self.warning("Seems to be a clean environment anyway") oldVersion = None if oldVersion: for e in list(os.environ.keys()): for p in ["WM_", "FOAM_"]: if e.find(p) == 0: del os.environ[e] break for p in [ "PATH", "DYLD_LIBRARY_PATH", "LD_LIBRARY_PATH", "MANPATH", "PV_PLUGIN_PATH" ]: if p in os.environ: lst = os.environ[p].split(":") os.environ[p] = ":".join( [l for l in lst if l.find(oldVersion) < 0]) if path.exists(cfgFile) and path.isfile(cfgFile) and os.access( cfgFile, os.R_OK): print_(" Using file " + cfgFile + " for loading the OpenFOAM environment") # Some more sanity check fileData = open(cfgFile).read() if not useCsh and "setenv" in fileData: self.error( " Error: Detecting 'setenv' instructions in this bash file. Please provide a configuration file compatible for bash, or use the --csh option." ) elif useCsh and "export" in fileData: self.error( " Error: Detecting 'export' instructions in this csh file. Please provide a configuration file compatible for csh." ) else: print_(" The configuration file seems ok") else: self.error(" Error: Cannot access file: " + cfgFile) # We choose not to run if the OpenFOAM environment is already loaded. # We obviously cannot diff from such an environment if os.getenv("WM_PROJECT") != None: self.error( """ Error: Cannot run with OpenFOAM environment variables already present You need to run this script from a clean environment""") # We choose not to run under the user 'root', simply because the HOME and USER environment # variables are usually very similar for this user, and this will cause problem later on # For instance, under Centos, for the super user: # HOME=/root # USER=root # Those two are very similar if os.getenv("USER") == "root": self.error( """ Error: You should not run this script from the 'root' account. Please invoke this pyFoam command from a plain OpenFOAM user account. Then, make the result file available to the super user (root) of the" destination host in order to install the modules configuration files.""" ) # Grab environment + aliases before the activation of OpenFOAM # We start from a minimal environment using 'env -i' shellCmd = 'bash -c' sourceCmd = '.' if useCsh: shellCmd = 'csh -c' sourceCmd = 'source' oldEnv = Set(execute(shellCmd + ' "env|sort"')) oldAliases = Set(execute(shellCmd + ' "alias|sort"')) # Grab environment + aliases after the activation of OpenFOAM newEnv = Set( execute(shellCmd + ' \"' + sourceCmd + ' ' + cfgFile + '; env|sort"')) newAliases = Set( execute(shellCmd + ' \"' + sourceCmd + ' ' + cfgFile + '; alias|sort"')) # Handling of environment variables # Memorize all the PATH environment variables for later processing oldPath = {} newPath = {} homeVar = "" userVar = "" for v in oldEnv: if v.find('=') > -1: key, value = v.split('=', 1) # Memorize PATH variables if key.find('PATH') > -1: oldPath[key] = value.replace('\n', '') # We remove any reference to PyFOAM in the old PATH env. variable # Since we run this script with PyFOAM, environment variables for PyFoam are # present in oldPath. If the configuration file for OpenFOAM requires PyFoam too, # we risk to lose these variables because they will be in common in both the # 'before' and 'after'environment. # This is ugly, but PyFOAM is just too useful to write this script with plain # python. # If you have installed PyFoam under a path where the string 'PyFoam' is not present, # you will simply have to manually add the necessary PyFoam environment variables # yourself in the module file. No biggie, oldPath[ key] = self.removeComponentWithSubstrFromPathEnvVar( oldPath[key], 'PyFoam') # Memorize HOME variable if key == 'HOME': homeVar = value.replace('\n', '') # Memorize USER variable if key == 'USER': userVar = value.replace('\n', '') # Using sets, it is trivial to remove common environment variables # and aliases between the two environments moduleEnv = newEnv - oldEnv moduleAliases = newAliases - oldAliases # Dictionary for environment variables envVar = {} # Dictionary for aliases aliasesVar = {} # Iterate through environment variables and store in dictionary for v in moduleEnv: if v.find('=') > -1: key, value = v.split('=') # Minor cleanup value = value.replace('\n', '') # Memorize PATH variables for later processing if key.find('PATH') > -1: newPath[key] = value else: envVar.setdefault(key, []).append(value) # Handle the PATH variables for v in newPath: if v in oldPath: # Cleanup old PATH components newPath[v] = self.removeComponentFromPathEnvVar( newPath[v], oldPath[v]) # Remove duplicate entries in PATH variable newPath[v] = self.uniqifyPathEnvVar(newPath[v]) envVar.setdefault(v, []).append(newPath[v]) # Iterate through aliases variables and store them in dictionary for v in moduleAliases: if v.find('=') > -1: key, value = v.split('=', 1) # Minor cleanup key = key.replace("alias ", "") value = value.replace('\n', "") value = value.replace('\'', "") aliasesVar.setdefault(key, []).append(value) # Generate module entries print_(" Generating modulefile: ", moduleFile) f = open(moduleFile, 'w') self.writeModuleFileHeader(f, envVar["WM_PROJECT_VERSION"][0]) self.writeModuleEnvironmentEntry(f, envVar, homeVar, userVar) self.writeModuleAliasEntry(f, aliasesVar, homeVar, userVar) self.writeModuleFileFooter(f) f.close() print_(" Done\n")
def prepare(self, sol, cName=None, overrideParameters=None): if cName == None: cName = sol.name if self.opts.onlyVariables: self.opts.verbose = True vals = {} vals["casePath"] = '"' + path.abspath(cName) + '"' vals["caseName"] = '"' + path.basename(path.abspath(cName)) + '"' vals["foamVersion"] = foamVersion() vals["foamFork"] = foamFork() if self.opts.verbose: print_("Looking for template values", cName) for f in self.opts.valuesDicts: if self.opts.verbose: print_("Reading values from", f) vals.update( ParsedParameterFile(f, noHeader=True, doMacroExpansion=True).getValueDict()) for v in self.opts.values: if self.opts.verbose: print_("Updating values", v) vals.update(eval(v)) if overrideParameters: vals.update(overrideParameters) if self.opts.verbose and len(vals) > 0: print_("\nUsed values\n") nameLen = max(len("Name"), max(*[len(k) for k in vals.keys()])) format = "%%%ds - %%s" % nameLen print_(format % ("Name", "Value")) print_("-" * 40) for k, v in sorted(iteritems(vals)): print_(format % (k, v)) print_("") elif self.opts.verbose: print_("\nNo values specified\n") if self.opts.onlyVariables: return if self.opts.doClear: if self.opts.verbose: print_("Clearing", cName) sol.clear(processor=True, pyfoam=True, vtk=True, removeAnalyzed=True, keepParallel=False, clearHistory=False, clearParameters=True, additional=["postProcessing"]) if self.opts.writeParameters: fName = path.join(cName, self.parameterOutFile) if self.opts.verbose: print_("Writing parameters to", fName) with WriteParameterFile(fName, noHeader=True) as w: w.content.update(vals, toString=True) w["foamVersion"] = vals["foamVersion"] w.writeFile() self.addToCaseLog(cName) for over in self.opts.overloadDirs: if self.opts.verbose: print_("Overloading files from", over) self.overloadDir(sol.name, over) zeroOrig = path.join(sol.name, "0.org") hasOrig = path.exists(zeroOrig) if not hasOrig: if self.opts.verbose: print_("Not going to clean '0'") self.opts.cleanDirectories.remove("0") if self.opts.doCopy: if hasOrig: if self.opts.verbose: print_("Found 0.org. Clearing 0") zeroDir = path.join(sol.name, "0") if path.exists(zeroDir): rmtree(zeroDir) elif self.opts.verbose: print_("No 0-directory") if self.opts.verbose: print_("") if self.opts.doTemplates: self.searchAndReplaceTemplates(sol.name, vals, self.opts.templateExt) if self.opts.verbose: print_("") if self.opts.doMeshCreate: if self.opts.meshCreateScript: scriptName = path.join(sol.name, self.opts.meshCreateScript) if not path.exists(scriptName): self.error("Script", scriptName, "does not exist") elif path.exists(path.join(sol.name, self.defaultMeshCreate)): scriptName = path.join(sol.name, self.defaultMeshCreate) else: scriptName = None if scriptName: if self.opts.verbose: print_("Executing", scriptName, "for mesh creation") if self.opts.verbose: echo = "Mesh: " else: echo = None result = "".join( execute([scriptName], workdir=sol.name, echo=echo)) open(scriptName + ".log", "w").write(result) else: if self.opts.verbose: print_( "No script for mesh creation found. Looking for 'blockMeshDict'" ) if sol.blockMesh() != "": if self.opts.verbose: print_(sol.blockMesh(), "found. Executing 'blockMesh'") bm = BasicRunner(argv=["blockMesh", "-case", sol.name]) bm.start() if not bm.runOK(): self.error("Problem with blockMesh") if self.opts.verbose: print_("") if self.opts.doCopy: self.copyOriginals(sol.name) if self.opts.verbose: print_("") if self.opts.doPostTemplates: self.searchAndReplaceTemplates(sol.name, vals, self.opts.postTemplateExt) if self.opts.verbose: print_("") if self.opts.doCaseSetup: if self.opts.caseSetupScript: scriptName = path.join(sol.name, self.opts.caseSetupScript) if not path.exists(scriptName): self.error("Script", scriptName, "does not exist") elif path.exists(path.join(sol.name, self.defaultCaseSetup)): scriptName = path.join(sol.name, self.defaultCaseSetup) else: scriptName = None if scriptName: if self.opts.verbose: print_("Executing", scriptName, "for case setup") if self.opts.verbose: echo = "Case:" else: echo = None result = "".join( execute([scriptName], workdir=sol.name, echo=echo)) open(scriptName + ".log", "w").write(result) else: if self.opts.verbose: print_("No script for case-setup found. Nothing done") if self.opts.verbose: print_("") if self.opts.doTemplateClean: if self.opts.verbose: print_("Clearing templates") for d in self.opts.cleanDirectories: for e in [self.opts.templateExt, self.opts.postTemplateExt]: self.cleanExtension(path.join(sol.name, d), e) if self.opts.verbose: print_("") if self.opts.verbose: print_("Case setup finished")
def prepare(self,sol, cName=None, overrideParameters=None, numberOfProcessors=None): """Do the actual preparing @param numberOfProcessors: If set this overrides the value set in the command line""" if cName==None: cName=sol.name if self.opts.onlyVariables: self.opts.verbose=True vals={} vals,self.metaData=self.getDefaultValues(cName) vals.update(self.addDictValues("System", "Automatically defined values", { "casePath" : '"'+path.abspath(cName)+'"', "caseName" : '"'+path.basename(path.abspath(cName))+'"', "foamVersion" : foamVersion(), "foamFork" : foamFork(), "numberOfProcessors" : numberOfProcessors if numberOfProcessors!=None else self.opts.numberOfProcessors })) if len(self.opts.extensionAddition)>0: vals.update(self.addDictValues("ExtensionAdditions", "Additional extensions to be processed", dict((e,True) for e in self.opts.extensionAddition))) valsWithDefaults=set(vals.keys()) self.info("Looking for template values",cName) for f in self.opts.valuesDicts: self.info("Reading values from",f) vals.update(ParsedParameterFile(f, noHeader=True, doMacroExpansion=True).getValueDict()) setValues={} for v in self.opts.values: self.info("Updating values",v) vals.update(eval(v)) setValues.update(eval(v)) if overrideParameters: vals.update(overrideParameters) unknownValues=set(vals.keys())-valsWithDefaults if len(unknownValues)>0: self.warning("Values for which no default was specified: "+ ", ".join(unknownValues)) if self.opts.verbose and len(vals)>0: print_("\nUsed values\n") nameLen=max(len("Name"), max(*[len(k) for k in vals.keys()])) format="%%%ds - %%s" % nameLen print_(format % ("Name","Value")) print_("-"*40) for k,v in sorted(iteritems(vals)): print_(format % (k,v)) print_("") else: self.info("\nNo values specified\n") self.checkCorrectOptions(vals) derivedScript=path.join(cName,self.opts.derivedParametersScript) if path.exists(derivedScript): self.info("Deriving variables in script",derivedScript) scriptText=open(derivedScript).read() glob={} oldVals=vals.copy() exec_(scriptText,glob,vals) added=[] changed=[] for k,v in iteritems(vals): if k not in oldVals: added.append(k) elif vals[k]!=oldVals[k]: changed.append(k) if len(changed)>0 and (not self.opts.allowDerivedChanges and not configuration().getboolean("PrepareCase","AllowDerivedChanges")): self.error(self.opts.derivedParametersScript, "changed values of"," ".join(changed), "\nTo allow this set --allow-derived-changes or the configuration item 'AllowDerivedChanges'") if len(added)>0: self.info("Added values:"," ".join(added)) if len(changed)>0: self.info("Changed values:"," ".join(changed)) if len(added)==0 and len(changed)==0: self.info("Nothing added or changed") else: self.info("No script",derivedScript,"for derived values") if self.opts.onlyVariables: return if self.opts.doClear: self.info("Clearing",cName) sol.clear(processor=True, pyfoam=True, vtk=True, removeAnalyzed=True, keepParallel=False, clearHistory=False, clearParameters=True, additional=["postProcessing"]) if self.opts.writeParameters: fName=path.join(cName,self.parameterOutFile) self.info("Writing parameters to",fName) with WriteParameterFile(fName,noHeader=True) as w: w.content.update(vals,toString=True) w["foamVersion"]=vals["foamVersion"] w.writeFile() if self.opts.writeReport: fName=path.join(cName,self.parameterOutFile+".rst") self.info("Writing report to",fName) with open(fName,"w") as w: helper=RestructuredTextHelper(defaultHeading=1) w.write(".. title:: "+self.__strip(vals["caseName"])+"\n") w.write(".. sectnum::\n") w.write(".. header:: "+self.__strip(vals["caseName"])+"\n") w.write(".. header:: "+time.asctime()+"\n") w.write(".. footer:: ###Page### / ###Total###\n\n") w.write("Parameters set in case directory "+ helper.literal(self.__strip(vals["casePath"]))+" at "+ helper.emphasis(time.asctime())+"\n\n") w.write(".. contents::\n\n") if len(self.opts.valuesDicts): w.write(helper.heading("Parameter files")) w.write("Parameters read from files\n\n") w.write(helper.enumerateList([helper.literal(f) for f in self.opts.valuesDicts])) w.write("\n") if len(setValues)>0: w.write(helper.heading("Overwritten parameters")) w.write("These parameters were set from the command line\n\n") w.write(helper.definitionList(setValues)) w.write("\n") w.write(helper.heading("Parameters with defaults")) w.write(self.makeReport(vals)) if len(unknownValues)>0: w.write(helper.heading("Unspecified parameters")) w.write("If these parameters are actually used then specify them in "+ helper.literal(self.defaultParameterFile)+"\n\n") tab=helper.table(True) for u in unknownValues: tab.addRow(u) tab.addItem("Value",vals[u]) w.write(str(tab)) self.addToCaseLog(cName) for over in self.opts.overloadDirs: self.info("Overloading files from",over) self.overloadDir(sol.name,over) zeroOrig=path.join(sol.name,"0.org") hasOrig=path.exists(zeroOrig) cleanZero=True if not hasOrig: self.info("Not going to clean '0'") self.opts.cleanDirectories.remove("0") cleanZero=False if self.opts.doCopy: if hasOrig: self.info("Found 0.org. Clearing 0") zeroDir=path.join(sol.name,"0") if path.exists(zeroDir): rmtree(zeroDir) else: self.info("No 0-directory") self.info("") else: cleanZero=False if self.opts.doTemplates: self.searchAndReplaceTemplates(sol.name, vals, self.opts.templateExt) self.info("") backupZeroDir=None if self.opts.doMeshCreate: if self.opts.meshCreateScript: scriptName=path.join(sol.name,self.opts.meshCreateScript) if not path.exists(scriptName): self.error("Script",scriptName,"does not exist") elif path.exists(path.join(sol.name,self.defaultMeshCreate)): scriptName=path.join(sol.name,self.defaultMeshCreate) else: scriptName=None if scriptName: self.info("Executing",scriptName,"for mesh creation") if self.opts.verbose: echo="Mesh: " else: echo=None result="".join(execute([scriptName],workdir=sol.name,echo=echo)) open(scriptName+".log","w").write(result) else: self.info("No script for mesh creation found. Looking for 'blockMeshDict'") if sol.blockMesh()!="": self.info(sol.blockMesh(),"found. Executing 'blockMesh'") bm=BasicRunner(argv=["blockMesh","-case",sol.name]) bm.start() if not bm.runOK(): self.error("Problem with blockMesh") for r in sol.regions(): self.info("Checking region",r) s=SolutionDirectory(sol.name,region=r, archive=None,paraviewLink=False) if s.blockMesh()!="": self.info(s.blockMesh(),"found. Executing 'blockMesh'") bm=BasicRunner(argv=["blockMesh","-case",sol.name, "-region",r]) bm.start() if not bm.runOK(): self.error("Problem with blockMesh") self.info("") if cleanZero and path.exists(zeroDir): self.warning("Mesh creation recreated 0-directory") if self.opts.keepZeroDirectoryFromMesh: backupZeroDir=zeroDir+".bakByPyFoam" self.info("Backing up",zeroDir,"to",backupZeroDir) move(zeroDir,backupZeroDir) else: self.info("Data in",zeroDir,"will be removed") if self.opts.doCopy: self.copyOriginals(sol.name) self.info("") if backupZeroDir: self.info("Copying backups from",backupZeroDir,"to",zeroDir) self.overloadDir(zeroDir,backupZeroDir) self.info("Removing backup",backupZeroDir) rmtree(backupZeroDir) if self.opts.doPostTemplates: self.searchAndReplaceTemplates(sol.name, vals, self.opts.postTemplateExt) self.info("") if self.opts.doCaseSetup: if self.opts.caseSetupScript: scriptName=path.join(sol.name,self.opts.caseSetupScript) if not path.exists(scriptName): self.error("Script",scriptName,"does not exist") elif path.exists(path.join(sol.name,self.defaultCaseSetup)): scriptName=path.join(sol.name,self.defaultCaseSetup) else: scriptName=None if scriptName: self.info("Executing",scriptName,"for case setup") if self.opts.verbose: echo="Case:" else: echo=None result="".join(execute([scriptName],workdir=sol.name,echo=echo)) open(scriptName+".log","w").write(result) else: self.info("No script for case-setup found. Nothing done") self.info("") if self.opts.doFinalTemplates: self.searchAndReplaceTemplates(sol.name, vals, self.opts.finalTemplateExt) if self.opts.doTemplateClean: self.info("Clearing templates") for d in self.opts.cleanDirectories: for e in [self.opts.templateExt, self.opts.postTemplateExt, self.opts.finalTemplateExt]: self.cleanExtension(path.join(sol.name,d),e) self.info("") self.info("Case setup finished")
def run(self): config = ConfigParser.ConfigParser() files = self.parser.getArgs() good = config.read(files) # will work with 2.4 # if len(good)!=len(files): # print_("Problem while trying to parse files",files) # print_("Only ",good," could be parsed") # sys.exit(-1) benchName = config.get("General", "name") if self.opts.nameAddition != None: benchName += "_" + self.opts.nameAddition if self.opts.foamVersion != None: benchName += "_v" + self.opts.foamVersion isParallel = config.getboolean("General", "parallel") lam = None if isParallel: nrCpus = config.getint("General", "nProcs") machineFile = config.get("General", "machines") if not path.exists(machineFile): self.error("Machine file ", machineFile, "needed for parallel run") lam = LAMMachine(machineFile, nr=nrCpus) if lam.cpuNr() > nrCpus: self.error("Wrong number of CPUs: ", lam.cpuNr()) print_("Running parallel on", lam.cpuNr(), "CPUs") if config.has_option("General", "casesDirectory"): casesDirectory = path.expanduser( config.get("General", "casesDirectory")) else: casesDirectory = foamTutorials() if not path.exists(casesDirectory): self.error("Directory", casesDirectory, "needed with the benchmark cases is missing") else: print_("Using cases from directory", casesDirectory) benchCases = [] config.remove_section("General") for sec in config.sections(): print_("Reading: ", sec) skipIt = False skipReason = "" if config.has_option(sec, "skip"): skipIt = config.getboolean(sec, "skip") skipReason = "Switched off in file" if self.opts.excases != None and not skipIt: for p in self.opts.excases: if fnmatch(sec, p): skipIt = True skipReason = "Switched off by pattern '" + p + "'" if self.opts.cases != None: for p in self.opts.cases: if fnmatch(sec, p): skipIt = False skipReason = "" if skipIt: print_("Skipping case ..... Reason:" + skipReason) continue sol = config.get(sec, "solver") cas = config.get(sec, "case") pre = eval(config.get(sec, "prepare")) preCon = [] if config.has_option(sec, "preControlDict"): preCon = eval(config.get(sec, "preControlDict")) con = eval(config.get(sec, "controlDict")) bas = config.getfloat(sec, "baseline") wei = config.getfloat(sec, "weight") add = [] if config.has_option(sec, "additional"): add = eval(config.get(sec, "additional")) print_("Adding: ", add) util = [] if config.has_option(sec, "utilities"): util = eval(config.get(sec, "utilities")) print_("Utilities: ", util) nr = 99999 if config.has_option(sec, "nr"): nr = eval(config.get(sec, "nr")) sp = None if config.has_option(sec, "blockSplit"): sp = eval(config.get(sec, "blockSplit")) toRm = [] if config.has_option(sec, "filesToRemove"): toRm = eval(config.get(sec, "filesToRemove")) setInit = [] if config.has_option(sec, "setInitial"): setInit = eval(config.get(sec, "setInitial")) parallelOK = False if config.has_option(sec, "parallelOK"): parallelOK = config.getboolean(sec, "parallelOK") deMet = ["metis"] if config.has_option(sec, "decomposition"): deMet = config.get(sec, "decomposition").split() if deMet[0] == "metis": pass elif deMet[0] == "simple": if len(deMet) < 2: deMet.append(0) else: deMet[1] = int(deMet[1]) else: print_("Unimplemented decomposition method", deMet[0], "switching to metis") deMet = ["metis"] if isParallel == False or parallelOK == True: if path.exists(path.join(casesDirectory, sol, cas)): benchCases.append( (nr, sec, sol, cas, pre, con, preCon, bas, wei, add, util, sp, toRm, setInit, deMet)) else: print_("Skipping", sec, "because directory", path.join(casesDirectory, sol, cas), "could not be found") else: print_("Skipping", sec, "because not parallel") benchCases.sort() parallelString = "" if isParallel: parallelString = ".cpus=" + str(nrCpus) resultFile = open( "Benchmark." + benchName + "." + uname()[1] + parallelString + ".results", "w") totalSpeedup = 0 minSpeedup = None maxSpeedup = None totalWeight = 0 runsOK = 0 currentEstimate = 1. print_("\nStart Benching\n") csv = CSVCollection("Benchmark." + benchName + "." + uname()[1] + parallelString + ".csv") # csvHeaders=["description","solver","case","caseDir","base", # "benchmark","machine","arch","cpus","os","version", # "wallclocktime","cputime","cputimeuser","cputimesystem","maxmemory","cpuusage","speedup"] for nr, description, solver, case, prepare, control, preControl, base, weight, additional, utilities, split, toRemove, setInit, decomposition in benchCases: # control.append( ("endTime",-2000) ) print_("Running Benchmark: ", description) print_("Solver: ", solver) print_("Case: ", case) caseName = solver + "_" + case + "_" + benchName + "." + uname( )[1] + ".case" print_("Short name: ", caseName) caseDir = caseName + ".runDir" csv["description"] = description csv["solver"] = solver csv["case"] = case csv["caseDir"] = caseDir csv["base"] = base csv["benchmark"] = benchName csv["machine"] = uname()[1] csv["arch"] = uname()[4] if lam == None: csv["cpus"] = 1 else: csv["cpus"] = lam.cpuNr() csv["os"] = uname()[0] csv["version"] = uname()[2] workDir = path.realpath(path.curdir) orig = SolutionDirectory(path.join(casesDirectory, solver, case), archive=None, paraviewLink=False) for a in additional + utilities: orig.addToClone(a) orig.cloneCase(path.join(workDir, caseDir)) if oldApp(): argv = [solver, workDir, caseDir] else: argv = [solver, "-case", path.join(workDir, caseDir)] run = BasicRunner(silent=True, argv=argv, logname="BenchRunning", lam=lam) runDir = run.getSolutionDirectory() controlFile = ParameterFile(runDir.controlDict()) for name, value in preControl: print_("Setting parameter", name, "to", value, "in controlDict") controlFile.replaceParameter(name, value) for rm in toRemove: fn = path.join(caseDir, rm) print_("Removing file", fn) remove(fn) for field, bc, val in setInit: print_("Setting", field, "on", bc, "to", val) SolutionFile(runDir.initialDir(), field).replaceBoundary(bc, val) oldDeltaT = controlFile.replaceParameter("deltaT", 0) for u in utilities: print_("Building utility ", u) execute("wmake 2>&1 >%s %s" % (path.join( caseDir, "BenchCompile." + u), path.join(caseDir, u))) print_("Preparing the case: ") if lam != None: prepare = prepare + [("decomposePar", "")] if decomposition[0] == "metis": lam.writeMetis( SolutionDirectory(path.join(workDir, caseDir))) elif decomposition[0] == "simple": lam.writeSimple( SolutionDirectory(path.join(workDir, caseDir)), decomposition[1]) if split: print_("Splitting the mesh:", split) bm = BlockMesh(runDir.blockMesh()) bm.refineMesh(split) for pre, post in prepare: print_("Doing ", pre, " ....") post = post.replace("%case%", caseDir) if oldApp(): args = string.split("%s %s %s %s" % (pre, workDir, caseDir, post)) else: args = string.split( "%s -case %s %s" % (pre, path.join(workDir, caseDir), post)) util = BasicRunner(silent=True, argv=args, logname="BenchPrepare_" + pre) util.start() controlFile.replaceParameter("deltaT", oldDeltaT) # control.append(("endTime",-1000)) for name, value in control: print_("Setting parameter", name, "to", value, "in controlDict") controlFile.replaceParameter(name, value) print_("Starting at ", asctime(localtime(time()))) print_( " Baseline is %f, estimated speedup %f -> estimated end at %s " % (base, currentEstimate, asctime(localtime(time() + base / currentEstimate)))) print_("Running the case ....") run.start() speedup = None cpuUsage = 0 speedupOut = -1 try: speedup = base / run.run.wallTime() cpuUsage = 100. * run.run.cpuTime() / run.run.wallTime() except ZeroDivisionError: print_("Division by Zero: ", run.run.wallTime()) if not run.runOK(): print_("\nWARNING!!!!") print_( "Run had a problem, not using the results. Check the log\n" ) speedup = None if speedup != None: speedupOut = speedup totalSpeedup += speedup * weight totalWeight += weight runsOK += 1 if maxSpeedup == None: maxSpeedup = speedup elif speedup > maxSpeedup: maxSpeedup = speedup if minSpeedup == None: minSpeedup = speedup elif speedup < minSpeedup: minSpeedup = speedup print_("Wall clock: ", run.run.wallTime()) print_("Speedup: ", speedup, " (Baseline: ", base, ")") print_("CPU Time: ", run.run.cpuTime()) print_("CPU Time User: "******"CPU Time System: ", run.run.cpuSystemTime()) print_("Memory: ", run.run.usedMemory()) print_("CPU Usage: %6.2f%%" % (cpuUsage)) csv["wallclocktime"] = run.run.wallTime() csv["cputime"] = run.run.cpuTime() csv["cputimeuser"] = run.run.cpuUserTime() csv["cputimesystem"] = run.run.cpuSystemTime() csv["maxmemory"] = run.run.usedMemory() csv["cpuusage"] = cpuUsage if speedup != None: csv["speedup"] = speedup else: csv["speedup"] = "##" csv.write() resultFile.write( "Case %s WallTime %g CPUTime %g UserTime %g SystemTime %g Memory %g MB Speedup %g\n" % (caseName, run.run.wallTime(), run.run.cpuTime(), run.run.cpuUserTime(), run.run.cpuSystemTime(), run.run.usedMemory(), speedupOut)) resultFile.flush() if speedup != None: currentEstimate = totalSpeedup / totalWeight if self.opts.removeCases: print_("Clearing case", end=" ") if speedup == None: print_("not ... because it failed") else: print_("completely") rmtree(caseDir, ignore_errors=True) print_() print_()