def copyOriginals(self, startDir):
"""Go recursivly through directories and copy foo.org to foo"""
self.info("Looking for originals in", startDir)
for f, t in self.listdir(startDir, self.opts.originalExt):
if f[0] == ".":
self.info("Skipping", f)
continue
src = path.join(startDir, f)
if t != None:
dst = path.join(startDir, t)
if path.exists(dst):
self.info("Replacing", dst, "with", src)
rmtree(dst)
else:
self.info("Copying", src, "to", dst)
copytree(src, dst, force=True)
elif path.isdir(src):
self.copyOriginals(src)
def copyOriginals(self,startDir):
"""Go recursivly through directories and copy foo.org to foo"""
self.info("Looking for originals in",startDir)
for f,t in self.listdir(startDir,self.opts.originalExt):
if f[0]==".":
self.info("Skipping",f)
continue
src=path.join(startDir,f)
if t!=None:
dst=path.join(startDir,t)
if path.exists(dst):
self.info("Replacing",dst,"with",src)
rmtree(dst)
else:
self.info("Copying",src,"to",dst)
copytree(src,dst,force=True)
elif path.isdir(src):
self.copyOriginals(src)
def copyOriginals(self, startDir):
"""Go recursivly through directories and copy foo.org to foo"""
if self.opts.verbose:
print_("Looking for originals in", startDir)
for f in listdir(startDir):
if f[0] == ".":
if self.opts.verbose:
print_("Skipping", f)
continue
src = path.join(startDir, f)
if path.splitext(f)[1] == self.opts.originalExt:
dst = path.join(startDir, path.splitext(f)[0])
if path.exists(dst):
if self.opts.verbose:
print_("Replacing", dst, "with", src)
rmtree(dst)
else:
if self.opts.verbose:
print_("Copying", src, "to", dst)
copytree(src, dst, force=True)
elif path.isdir(src):
self.copyOriginals(src)
def run(self):
decomposeParWithRegion=(foamVersion()>=(1,6))
if self.opts.keeppseudo and (not self.opts.regions and self.opts.region==None):
warning("Option --keep-pseudocases only makes sense for multi-region-cases")
if decomposeParWithRegion and self.opts.keeppseudo:
warning("Option --keep-pseudocases doesn't make sense since OpenFOAM 1.6 because decomposePar supports regions")
nr=int(self.parser.getArgs()[1])
if nr<2:
error("Number of processors",nr,"too small (at least 2)")
case=path.abspath(self.parser.getArgs()[0])
method=self.opts.method
result={}
result["numberOfSubdomains"]=nr
result["method"]=method
coeff={}
result[method+"Coeffs"]=coeff
if self.opts.globalFaceZones!=None:
try:
fZones=eval(self.opts.globalFaceZones)
except SyntaxError:
fZones=FoamStringParser(
self.opts.globalFaceZones,
listDict=True
).data
result["globalFaceZones"]=fZones
if method in ["metis","scotch","parMetis"]:
if self.opts.processorWeights!=None:
weigh=eval(self.opts.processorWeights)
if nr!=len(weigh):
error("Number of processors",nr,"and length of",weigh,"differ")
coeff["processorWeights"]=weigh
elif method=="manual":
if self.opts.dataFile==None:
error("Missing required option dataFile")
else:
coeff["dataFile"]="\""+self.opts.dataFile+"\""
elif method=="simple" or method=="hierarchical":
if self.opts.n==None or self.opts.delta==None:
error("Missing required option n or delta")
n=eval(self.opts.n)
if len(n)!=3:
error("Needs to be three elements, not",n)
if nr!=n[0]*n[1]*n[2]:
error("Subdomains",n,"inconsistent with processor number",nr)
coeff["n"]="(%d %d %d)" % (n[0],n[1],n[2])
coeff["delta"]=float(self.opts.delta)
if method=="hierarchical":
if self.opts.order==None:
error("Missing reuired option order")
if len(self.opts.order)!=3:
error("Order needs to be three characters")
coeff["order"]=self.opts.order
else:
error("Method",method,"not yet implementes")
gen=FoamFileGenerator(result)
if self.opts.test:
print_(str(gen))
return -1
else:
f=open(path.join(case,"system","decomposeParDict"),"w")
writeDictionaryHeader(f)
f.write(str(gen))
f.close()
if self.opts.clear:
print_("Clearing processors")
for p in glob(path.join(case,"processor*")):
print_("Removing",p)
rmtree(p,ignore_errors=True)
self.checkAndCommit(SolutionDirectory(case,archive=None))
if self.opts.doDecompose:
if self.opts.region:
regionNames=self.opts.region[:]
while True:
try:
i=regionNames.index("region0")
regionNames[i]=None
except ValueError:
break
else:
regionNames=[None]
regions=None
sol=SolutionDirectory(case)
if not decomposeParWithRegion:
if self.opts.regions or self.opts.region!=None:
print_("Building Pseudocases")
regions=RegionCases(sol,clean=True,processorDirs=False)
if self.opts.regions:
regionNames=sol.getRegions(defaultRegion=True)
for theRegion in regionNames:
theCase=path.normpath(case)
if theRegion!=None and not decomposeParWithRegion:
theCase+="."+theRegion
if oldApp():
argv=[self.opts.decomposer,".",theCase]
else:
argv=[self.opts.decomposer,"-case",theCase]
if foamVersion()>=(2,0) and not self.opts.doFunctionObjects:
argv+=["-noFunctionObjects"]
if theRegion!=None and decomposeParWithRegion:
argv+=["-region",theRegion]
f=open(path.join(case,"system",theRegion,"decomposeParDict"),"w")
writeDictionaryHeader(f)
f.write(str(gen))
f.close()
self.setLogname(default="Decomposer",useApplication=False)
run=UtilityRunner(argv=argv,
silent=self.opts.progress or self.opts.silent,
logname=self.opts.logname,
compressLog=self.opts.compress,
server=self.opts.server,
noLog=self.opts.noLog,
logTail=self.opts.logTail,
echoCommandLine=self.opts.echoCommandPrefix,
jobId=self.opts.jobId)
run.start()
if theRegion!=None and not decomposeParWithRegion:
print_("Syncing into master case")
regions.resync(theRegion)
if regions!=None and not decomposeParWithRegion:
if not self.opts.keeppseudo:
print_("Removing pseudo-regions")
regions.cleanAll()
else:
for r in sol.getRegions():
if r not in regionNames:
regions.clean(r)
if self.opts.doConstantLinks:
print_("Adding symlinks in the constant directories")
constPath=path.join(case,"constant")
for f in listdir(constPath):
srcExpr=path.join(path.pardir,path.pardir,"constant",f)
for p in range(nr):
dest=path.join(case,"processor%d"%p,"constant",f)
if not path.exists(dest):
symlink(srcExpr,dest)
self.addToCaseLog(case)
def run(self):
config=ConfigParser.ConfigParser()
files=self.parser.getArgs()
good=config.read(files)
# will work with 2.4
# if len(good)!=len(files):
# print "Problem while trying to parse files",files
# print "Only ",good," could be parsed"
# sys.exit(-1)
benchName=config.get("General","name")
if self.opts.nameAddition!=None:
benchName+="_"+self.opts.nameAddition
if self.opts.foamVersion!=None:
benchName+="_v"+self.opts.foamVersion
isParallel=config.getboolean("General","parallel")
lam=None
if isParallel:
nrCpus=config.getint("General","nProcs")
machineFile=config.get("General","machines")
if not path.exists(machineFile):
self.error("Machine file ",machineFile,"needed for parallel run")
lam=LAMMachine(machineFile,nr=nrCpus)
if lam.cpuNr()>nrCpus:
self.error("Wrong number of CPUs: ",lam.cpuNr())
print "Running parallel on",lam.cpuNr(),"CPUs"
if config.has_option("General","casesDirectory"):
casesDirectory=path.expanduser(config.get("General","casesDirectory"))
else:
casesDirectory=foamTutorials()
if not path.exists(casesDirectory):
self.error("Directory",casesDirectory,"needed with the benchmark cases is missing")
else:
print "Using cases from directory",casesDirectory
benchCases=[]
config.remove_section("General")
for sec in config.sections():
print "Reading: ",sec
skipIt=False
skipReason=""
if config.has_option(sec,"skip"):
skipIt=config.getboolean(sec,"skip")
skipReason="Switched off in file"
if self.opts.excases!=None and not skipIt:
for p in self.opts.excases:
if fnmatch(sec,p):
skipIt=True
skipReason="Switched off by pattern '"+p+"'"
if self.opts.cases!=None:
for p in self.opts.cases:
if fnmatch(sec,p):
skipIt=False
skipReason=""
if skipIt:
print "Skipping case ..... Reason:"+skipReason
continue
sol=config.get(sec,"solver")
cas=config.get(sec,"case")
pre=eval(config.get(sec,"prepare"))
preCon=[]
if config.has_option(sec,"preControlDict"):
preCon=eval(config.get(sec,"preControlDict"))
con=eval(config.get(sec,"controlDict"))
bas=config.getfloat(sec,"baseline")
wei=config.getfloat(sec,"weight")
add=[]
if config.has_option(sec,"additional"):
add=eval(config.get(sec,"additional"))
print "Adding: ", add
util=[]
if config.has_option(sec,"utilities"):
util=eval(config.get(sec,"utilities"))
print "Utilities: ", util
nr=99999
if config.has_option(sec,"nr"):
nr=eval(config.get(sec,"nr"))
sp=None
if config.has_option(sec,"blockSplit"):
sp=eval(config.get(sec,"blockSplit"))
toRm=[]
if config.has_option(sec,"filesToRemove"):
toRm=eval(config.get(sec,"filesToRemove"))
setInit=[]
if config.has_option(sec,"setInitial"):
setInit=eval(config.get(sec,"setInitial"))
parallelOK=False
if config.has_option(sec,"parallelOK"):
parallelOK=config.getboolean(sec,"parallelOK")
deMet=["metis"]
if config.has_option(sec,"decomposition"):
deMet=config.get(sec,"decomposition").split()
if deMet[0]=="metis":
pass
elif deMet[0]=="simple":
if len(deMet)<2:
deMet.append(0)
else:
deMet[1]=int(deMet[1])
else:
print "Unimplemented decomposition method",deMet[0],"switching to metis"
deMet=["metis"]
if isParallel==False or parallelOK==True:
if path.exists(path.join(casesDirectory,sol,cas)):
benchCases.append( (nr,sec,sol,cas,pre,con,preCon,bas,wei,add,util,sp,toRm,setInit,deMet) )
else:
print "Skipping",sec,"because directory",path.join(casesDirectory,sol,cas),"could not be found"
else:
print "Skipping",sec,"because not parallel"
benchCases.sort()
parallelString=""
if isParallel:
parallelString=".cpus="+str(nrCpus)
resultFile=open("Benchmark."+benchName+"."+uname()[1]+parallelString+".results","w")
totalSpeedup=0
minSpeedup=None
maxSpeedup=None
totalWeight =0
runsOK=0
currentEstimate = 1.
print "\nStart Benching\n"
csv=CSVCollection("Benchmark."+benchName+"."+uname()[1]+parallelString+".csv")
# csvHeaders=["description","solver","case","caseDir","base",
# "benchmark","machine","arch","cpus","os","version",
# "wallclocktime","cputime","cputimeuser","cputimesystem","maxmemory","cpuusage","speedup"]
for nr,description,solver,case,prepare,control,preControl,base,weight,additional,utilities,split,toRemove,setInit,decomposition in benchCases:
# control.append( ("endTime",-2000) )
print "Running Benchmark: ",description
print "Solver: ",solver
print "Case: ",case
caseName=solver+"_"+case+"_"+benchName+"."+uname()[1]+".case"
print "Short name: ",caseName
caseDir=caseName+".runDir"
csv["description"]=description
csv["solver"]=solver
csv["case"]=case
csv["caseDir"]=caseDir
csv["base"]=base
csv["benchmark"]=benchName
csv["machine"]=uname()[1]
csv["arch"]=uname()[4]
if lam==None:
csv["cpus"]=1
else:
csv["cpus"]=lam.cpuNr()
csv["os"]=uname()[0]
csv["version"]=uname()[2]
workDir=path.realpath(path.curdir)
orig=SolutionDirectory(path.join(casesDirectory,solver,case),
archive=None,
paraviewLink=False)
for a in additional+utilities:
orig.addToClone(a)
orig.cloneCase(path.join(workDir,caseDir))
if oldApp():
argv=[solver,workDir,caseDir]
else:
argv=[solver,"-case",path.join(workDir,caseDir)]
run=BasicRunner(silent=True,argv=argv,logname="BenchRunning",lam=lam)
runDir=run.getSolutionDirectory()
controlFile=ParameterFile(runDir.controlDict())
for name,value in preControl:
print "Setting parameter",name,"to",value,"in controlDict"
controlFile.replaceParameter(name,value)
for rm in toRemove:
fn=path.join(caseDir,rm)
print "Removing file",fn
remove(fn)
for field,bc,val in setInit:
print "Setting",field,"on",bc,"to",val
SolutionFile(runDir.initialDir(),field).replaceBoundary(bc,val)
oldDeltaT=controlFile.replaceParameter("deltaT",0)
for u in utilities:
print "Building utility ",u
execute("wmake 2>&1 >%s %s" % (path.join(caseDir,"BenchCompile."+u),path.join(caseDir,u)))
print "Preparing the case: "
if lam!=None:
prepare=prepare+[("decomposePar","")]
if decomposition[0]=="metis":
lam.writeMetis(SolutionDirectory(path.join(workDir,caseDir)))
elif decomposition[0]=="simple":
lam.writeSimple(SolutionDirectory(path.join(workDir,caseDir)),decomposition[1])
if split:
print "Splitting the mesh:",split
bm=BlockMesh(runDir.blockMesh())
bm.refineMesh(split)
for pre,post in prepare:
print "Doing ",pre," ...."
post=post.replace("%case%",caseDir)
if oldApp():
args=string.split("%s %s %s %s" % (pre,workDir,caseDir,post))
else:
args=string.split("%s -case %s %s" % (pre,path.join(workDir,caseDir),post))
util=BasicRunner(silent=True,argv=args,logname="BenchPrepare_"+pre)
util.start()
controlFile.replaceParameter("deltaT",oldDeltaT)
# control.append(("endTime",-1000))
for name,value in control:
print "Setting parameter",name,"to",value,"in controlDict"
controlFile.replaceParameter(name,value)
print "Starting at ",asctime(localtime(time()))
print " Baseline is %f, estimated speedup %f -> estimated end at %s " % (base,currentEstimate,asctime(localtime(time()+base/currentEstimate)))
print "Running the case ...."
run.start()
speedup=None
cpuUsage=0
speedupOut=-1
try:
speedup=base/run.run.wallTime()
cpuUsage=100.*run.run.cpuTime()/run.run.wallTime()
except ZeroDivisionError:
print "Division by Zero: ",run.run.wallTime()
if not run.runOK():
print "\nWARNING!!!!"
print "Run had a problem, not using the results. Check the log\n"
speedup=None
if speedup!=None:
speedupOut=speedup
totalSpeedup+=speedup*weight
totalWeight +=weight
runsOK+=1
if maxSpeedup==None:
maxSpeedup=speedup
elif speedup>maxSpeedup:
maxSpeedup=speedup
if minSpeedup==None:
minSpeedup=speedup
elif speedup<minSpeedup:
minSpeedup=speedup
print "Wall clock: ",run.run.wallTime()
print "Speedup: ",speedup," (Baseline: ",base,")"
print "CPU Time: ",run.run.cpuTime()
print "CPU Time User: "******"CPU Time System: ",run.run.cpuSystemTime()
print "Memory: ",run.run.usedMemory()
print "CPU Usage: %6.2f%%" % (cpuUsage)
csv["wallclocktime"]=run.run.wallTime()
csv["cputime"]=run.run.cpuTime()
csv["cputimeuser"]=run.run.cpuUserTime()
csv["cputimesystem"]=run.run.cpuSystemTime()
csv["maxmemory"]=run.run.usedMemory()
csv["cpuusage"]=cpuUsage
if speedup!=None:
csv["speedup"]=speedup
else:
csv["speedup"]="##"
csv.write()
resultFile.write("Case %s WallTime %g CPUTime %g UserTime %g SystemTime %g Memory %g MB Speedup %g\n" %(caseName,run.run.wallTime(),run.run.cpuTime(),run.run.cpuUserTime(),run.run.cpuSystemTime(),run.run.usedMemory(),speedupOut))
resultFile.flush()
if speedup!=None:
currentEstimate=totalSpeedup/totalWeight
if self.opts.removeCases:
print "Clearing case",
if speedup==None:
print "not ... because it failed"
else:
print "completely"
rmtree(caseDir,ignore_errors=True)
print
print
if lam!=None:
lam.stop()
print "Total Speedup: ",currentEstimate," ( ",totalSpeedup," / ",totalWeight, " ) Range: [",minSpeedup,",",maxSpeedup,"]"
print runsOK,"of",len(benchCases),"ran OK"
resultFile.write("Total Speedup: %g\n" % (currentEstimate))
if minSpeedup and maxSpeedup:
resultFile.write("Range: [ %g , %g ]\n" % (minSpeedup,maxSpeedup))
resultFile.close()
def clean(self,region):
rmtree(self.master.name+"."+region,ignore_errors=True)
def __init__(self,
argv=None,
silent=False,
logname=None,
compressLog=False,
lam=None,
server=False,
restart=False,
noLog=False,
logTail=None,
remark=None,
jobId=None,
parameters=None,
writeState=True,
echoCommandLine=None):
""":param argv: list with the tokens that are the command line
if not set the standard command line is used
:param silent: if True no output is sent to stdout
:param logname: name of the logfile
:param compressLog: Compress the logfile into a gzip
:param lam: Information about a parallel run
:param server: Whether or not to start the network-server
:type lam: PyFoam.Execution.ParallelExecution.LAMMachine
:param noLog: Don't output a log file
:param logTail: only the last lines of the log should be written
:param remark: User defined remark about the job
:param parameters: User defined dictionary with parameters for
documentation purposes
:param jobId: Job ID of the controlling system (Queueing system)
:param writeState: Write the state to some files in the case
:param echoCommandLine: Prefix that is printed with the command line. If unset nothing is printed
"""
if sys.version_info < (2, 3):
# Python 2.2 does not have the capabilities for the Server-Thread
if server:
warning(
"Can not start server-process because Python-Version is too old"
)
server = False
if argv == None:
self.argv = sys.argv[1:]
else:
self.argv = argv
if oldApp():
self.dir = path.join(self.argv[1], self.argv[2])
if self.argv[2][-1] == path.sep:
self.argv[2] = self.argv[2][:-1]
else:
self.dir = path.curdir
if "-case" in self.argv:
self.dir = self.argv[self.argv.index("-case") + 1]
logname = calcLogname(logname, argv)
try:
sol = self.getSolutionDirectory()
except OSError:
e = sys.exc_info()[1] # compatible with 2.x and 3.x
error("Solution directory", self.dir,
"does not exist. No use running. Problem:", e)
self.echoCommandLine = echoCommandLine
self.silent = silent
self.lam = lam
self.origArgv = self.argv
self.writeState = writeState
self.__lastLastSeenWrite = 0
self.__lastNowTimeWrite = 0
if self.lam != None:
self.argv = lam.buildMPIrun(self.argv)
if config().getdebug("ParallelExecution"):
debug("Command line:", " ".join(self.argv))
self.cmd = " ".join(self.argv)
foamLogger().info("Starting: " + self.cmd + " in " +
path.abspath(path.curdir))
self.logFile = path.join(self.dir, logname + ".logfile")
isRestart, restartnr, restartName, lastlog = findRestartFiles(
self.logFile, sol)
if restartName:
self.logFile = restartName
if not isRestart:
from os import unlink
from glob import glob
for g in glob(self.logFile + ".restart*"):
if path.isdir(g):
rmtree(g)
else:
unlink(g)
self.noLog = noLog
self.logTail = logTail
if self.logTail:
if self.noLog:
warning("Log tail", self.logTail,
"and no-log specified. Using logTail")
self.noLog = True
self.lastLines = []
self.compressLog = compressLog
if self.compressLog:
self.logFile += ".gz"
self.fatalError = False
self.fatalFPE = False
self.fatalStackdump = False
self.endSeen = False
self.warnings = 0
self.started = False
self.isRestarted = False
if restart:
self.controlDict = ParameterFile(path.join(self.dir, "system",
"controlDict"),
backup=True)
self.controlDict.replaceParameter("startFrom", "latestTime")
self.isRestarted = True
else:
self.controlDict = None
self.run = FoamThread(self.cmd, self)
self.server = None
if server:
self.server = FoamServer(run=self.run, master=self)
self.server.setDaemon(True)
self.server.start()
try:
IP, PID, Port = self.server.info()
f = open(path.join(self.dir, "PyFoamServer.info"), "w")
print_(IP, PID, Port, file=f)
f.close()
except AttributeError:
warning(
"There seems to be a problem with starting the server:",
self.server, "with attributes", dir(self.server))
self.server = None
self.createTime = None
self.nowTime = None
self.startTimestamp = time()
self.stopMe = False
self.writeRequested = False
self.endTriggers = []
self.lastLogLineSeen = None
self.lastTimeStepSeen = None
self.remark = remark
self.jobId = jobId
self.data = {"lines": 0} # self.data={"lines":0L}
self.data["logfile"] = self.logFile
self.data["casefullname"] = path.abspath(self.dir)
self.data["casename"] = path.basename(path.abspath(self.dir))
self.data["solver"] = path.basename(self.argv[0])
self.data["solverFull"] = self.argv[0]
self.data["commandLine"] = self.cmd
self.data["hostname"] = uname()[1]
if remark:
self.data["remark"] = remark
else:
self.data["remark"] = "No remark given"
if jobId:
self.data["jobId"] = jobId
parameterFile = sol.getParametersFromFile()
if len(parameterFile):
self.data["parameters"] = {}
for k, v in parameterFile.items():
self.data["parameters"][k] = makePrimitiveString(v)
if parameters:
if "parameters" not in self.data:
self.data["parameters"] = {}
self.data["parameters"].update(parameters)
self.data["starttime"] = asctime()
def run(self):
decomposeParWithRegion = (foamVersion() >= (1, 6))
if self.opts.keeppseudo and (not self.opts.regions
and self.opts.region == None):
warning(
"Option --keep-pseudocases only makes sense for multi-region-cases"
)
if decomposeParWithRegion and self.opts.keeppseudo:
warning(
"Option --keep-pseudocases doesn't make sense since OpenFOAM 1.6 because decomposePar supports regions"
)
nr = int(self.parser.getArgs()[1])
if nr < 2:
error("Number of processors", nr, "too small (at least 2)")
case = path.abspath(self.parser.getArgs()[0])
method = self.opts.method
result = {}
result["numberOfSubdomains"] = nr
result["method"] = method
coeff = {}
result[method + "Coeffs"] = coeff
if self.opts.globalFaceZones != None:
try:
fZones = eval(self.opts.globalFaceZones)
except SyntaxError:
fZones = FoamStringParser(self.opts.globalFaceZones,
listDict=True).data
result["globalFaceZones"] = fZones
if method in ["metis", "scotch", "parMetis"]:
if self.opts.processorWeights != None:
weigh = eval(self.opts.processorWeights)
if nr != len(weigh):
error("Number of processors", nr, "and length of", weigh,
"differ")
coeff["processorWeights"] = weigh
elif method == "manual":
if self.opts.dataFile == None:
error("Missing required option dataFile")
else:
coeff["dataFile"] = "\"" + self.opts.dataFile + "\""
elif method == "simple" or method == "hierarchical":
if self.opts.n == None or self.opts.delta == None:
error("Missing required option n or delta")
n = eval(self.opts.n)
if len(n) != 3:
error("Needs to be three elements, not", n)
if nr != n[0] * n[1] * n[2]:
error("Subdomains", n, "inconsistent with processor number",
nr)
coeff["n"] = "(%d %d %d)" % (n[0], n[1], n[2])
coeff["delta"] = float(self.opts.delta)
if method == "hierarchical":
if self.opts.order == None:
error("Missing reuired option order")
if len(self.opts.order) != 3:
error("Order needs to be three characters")
coeff["order"] = self.opts.order
else:
error("Method", method, "not yet implementes")
gen = FoamFileGenerator(result)
if self.opts.test:
print_(str(gen))
return -1
else:
f = open(path.join(case, "system", "decomposeParDict"), "w")
writeDictionaryHeader(f)
f.write(str(gen))
f.close()
if self.opts.clear:
print_("Clearing processors")
for p in glob(path.join(case, "processor*")):
print_("Removing", p)
rmtree(p, ignore_errors=True)
def __init__(self,
argv=None,
silent=False,
logname=None,
compressLog=False,
lam=None,
server=False,
restart=False,
noLog=False,
logTail=None,
remark=None,
jobId=None,
parameters=None,
writeState=True,
echoCommandLine=None):
""":param argv: list with the tokens that are the command line
if not set the standard command line is used
:param silent: if True no output is sent to stdout
:param logname: name of the logfile
:param compressLog: Compress the logfile into a gzip
:param lam: Information about a parallel run
:param server: Whether or not to start the network-server
:type lam: PyFoam.Execution.ParallelExecution.LAMMachine
:param noLog: Don't output a log file
:param logTail: only the last lines of the log should be written
:param remark: User defined remark about the job
:param parameters: User defined dictionary with parameters for
documentation purposes
:param jobId: Job ID of the controlling system (Queueing system)
:param writeState: Write the state to some files in the case
:param echoCommandLine: Prefix that is printed with the command line. If unset nothing is printed
"""
if sys.version_info < (2,3):
# Python 2.2 does not have the capabilities for the Server-Thread
if server:
warning("Can not start server-process because Python-Version is too old")
server=False
if argv==None:
self.argv=sys.argv[1:]
else:
self.argv=argv
if oldApp():
self.dir=path.join(self.argv[1],self.argv[2])
if self.argv[2][-1]==path.sep:
self.argv[2]=self.argv[2][:-1]
else:
self.dir=path.curdir
if "-case" in self.argv:
self.dir=self.argv[self.argv.index("-case")+1]
logname=calcLogname(logname,argv)
try:
sol=self.getSolutionDirectory()
except OSError:
e = sys.exc_info()[1] # compatible with 2.x and 3.x
error("Solution directory",self.dir,"does not exist. No use running. Problem:",e)
self.echoCommandLine=echoCommandLine
self.silent=silent
self.lam=lam
self.origArgv=self.argv
self.writeState=writeState
self.__lastLastSeenWrite=0
self.__lastNowTimeWrite=0
if self.lam!=None:
self.argv=lam.buildMPIrun(self.argv)
if config().getdebug("ParallelExecution"):
debug("Command line:"," ".join(self.argv))
self.cmd=" ".join(self.argv)
foamLogger().info("Starting: "+self.cmd+" in "+path.abspath(path.curdir))
self.logFile=path.join(self.dir,logname+".logfile")
isRestart,restartnr,restartName,lastlog=findRestartFiles(self.logFile,sol)
if restartName:
self.logFile=restartName
if not isRestart:
from os import unlink
from glob import glob
for g in glob(self.logFile+".restart*"):
if path.isdir(g):
rmtree(g)
else:
unlink(g)
self.noLog=noLog
self.logTail=logTail
if self.logTail:
if self.noLog:
warning("Log tail",self.logTail,"and no-log specified. Using logTail")
self.noLog=True
self.lastLines=[]
self.compressLog=compressLog
if self.compressLog:
self.logFile+=".gz"
self.fatalError=False
self.fatalFPE=False
self.fatalStackdump=False
self.endSeen=False
self.warnings=0
self.started=False
self.isRestarted=False
if restart:
self.controlDict=ParameterFile(path.join(self.dir,"system","controlDict"),backup=True)
self.controlDict.replaceParameter("startFrom","latestTime")
self.isRestarted=True
else:
self.controlDict=None
self.run=FoamThread(self.cmd,self)
self.server=None
if server:
self.server=FoamServer(run=self.run,master=self)
self.server.setDaemon(True)
self.server.start()
try:
IP,PID,Port=self.server.info()
f=open(path.join(self.dir,"PyFoamServer.info"),"w")
print_(IP,PID,Port,file=f)
f.close()
except AttributeError:
warning("There seems to be a problem with starting the server:",self.server,"with attributes",dir(self.server))
self.server=None
self.createTime=None
self.nowTime=None
self.startTimestamp=time()
self.stopMe=False
self.writeRequested=False
self.endTriggers=[]
self.lastLogLineSeen=None
self.lastTimeStepSeen=None
self.remark=remark
self.jobId=jobId
self.data={"lines":0} # self.data={"lines":0L}
self.data["logfile"]=self.logFile
self.data["casefullname"]=path.abspath(self.dir)
self.data["casename"]=path.basename(path.abspath(self.dir))
self.data["solver"]=path.basename(self.argv[0])
self.data["solverFull"]=self.argv[0]
self.data["commandLine"]=self.cmd
self.data["hostname"]=uname()[1]
if remark:
self.data["remark"]=remark
else:
self.data["remark"]="No remark given"
if jobId:
self.data["jobId"]=jobId
parameterFile=sol.getParametersFromFile()
if len(parameterFile):
self.data["parameters"]={}
for k,v in parameterFile.items():
self.data["parameters"][k]=makePrimitiveString(v)
if parameters:
if "parameters" not in self.data:
self.data["parameters"]={}
self.data["parameters"].update(parameters)
self.data["starttime"]=asctime()
def run(self):
fName=self.parser.getArgs()[0]
dom=parse(fName)
doc=dom.documentElement
if doc.tagName!='comparator':
error("Wrong root-element",doc.tagName,"Expected: 'comparator'")
self.data=ComparatorData(doc)
purge=False
if doc.hasAttribute('purge'):
purge=eval(doc.getAttribute('purge'))
if self.opts.purge:
purge=self.opts.purge
if self.opts.nopurge:
purge=False
steady=False
if doc.hasAttribute('steady'):
steady=eval(doc.getAttribute('steady'))
if self.opts.steady:
purge=self.opts.steady
print_(" Parameters read OK ")
print_()
aLog=open(self.data.id+".overview","w")
csv=CSVCollection(self.data.id+".csv")
rDir=self.data.id+".results"
rmtree(rDir)
mkdir(rDir)
calculated=0
format="%%0%dd" % len(str(len(self.data)))
for i in range(len(self.data)):
runID=(format % i)
print_(runID,end=" ",file=aLog)
csv["ID"]=runID
use,para=self.data[i]
para["template"]=self.data.template
para["extension"]=self.data.extension
para["id"]=self.data.id
if use:
calculated+=1
print_("Executing Variation",i+1,"of",len(self.data),end=" ")
if calculated!=i+1:
print_("(",calculated,"actually calculated)")
else:
print_()
print_("Parameters:",end=" ")
for k,v in iteritems(para):
print_("%s='%s' " % (k,v),end=" ")
if v.find(" ")>=0 or v.find("\t")>=0:
v="'"+v+"'"
print_(v,end=" ",file=aLog)
csv[k]=v
print_()
if not use:
print_("Skipping because not all conditions are satisfied")
csv.clear()
print_()
continue
cName=("%s."+format) % (self.data.id, i)
log=open(cName+".log","w")
para["case"]=cName
print_("Case-directory:",cName)
para["results"]=path.join(rDir,runID)
print_("Results directory:",para["results"])
mkdir(para["results"])
if path.exists(cName):
if self.opts.removeOld:
print_(" Removing old case-directory")
rmtree(cName)
else:
error("Case-directory",cName,"exists")
print_(" copying template")
out=copytree(self.data.template,cName)
print_("---- Copying",file=log)
for l in out:
print_(l,end=" ",file=log)
print_(" preparing")
ok,erg=self.data.prep.execute(para,log)
print_(ok,end=" ",file=aLog)
csv["prepare OK"]=ok
for i in range(len(erg)):
print_(erg[i],end=" ",file=aLog)
csv["Prepare %02d" % i]=erg[i]
aLog.flush()
if self.opts.test:
print_(" Skipping execution")
else:
print_(" running the solver")
sys.stdout.flush()
if steady:
runnerClass=ConvergenceRunner
else:
runnerClass=AnalyzedRunner
run=runnerClass(BoundingLogAnalyzer(doTimelines=True,progress=True),
argv=[self.data.solver,".",cName],
silent=True,
lam=Command.parallel,
server=self.opts.server)
run.start()
ok=run.runOK()
if ok:
print_(" executed OK")
else:
print_(" fatal error")
for aName in run.listAnalyzers():
a=run.getAnalyzer(aName)
if 'titles' in dir(a):
for tit in a.lines.getValueNames():
t,v=a.getTimeline(tit)
if len(v)>0:
para["result_"+aName+"_"+tit]=v[-1]
print_(run.runOK(),run.lastTime(),run.run.wallTime(),end=" ",file=aLog)
csv["Run OK"]=run.runOK()
csv["End Time"]=run.lastTime()
csv["Wall Time"]=run.run.wallTime()
csv["Wall Time (Foam)"]=run.totalClockTime()
csv["CPU Time"]=run.totalCpuTime()
csv["Wall Time First Step"]=run.firstClockTime()
csv["CPU Time First Step"]=run.firstCpuTime()
para["endTime"]=run.lastTime()
para["runlog"]=run.logFile
if self.opts.showDict:
print_(para)
print_(" evaluating results")
ok,erg=self.data.post.execute(para,log)
if Command.parallel!=None:
print_(" Stoping LAM")
Command.parallel.stop()
Command.parallel=None
if ok:
print_(" Evaluation OK",end=" ")
else:
print_(" Evaluation failed",end=" ")
if len(erg)>0:
print_(":",erg,end=" ")
print_()
print_(ok,end=" ",file=aLog)
for i in range(len(erg)):
print_(erg[i],end=" ",file=aLog)
csv["Post %02d" % i]=erg[i]
if purge:
print_(" removing the case-directory")
out=rmtree(cName)
print_("---- Removing",file=log)
for l in out:
print_(l,end=" ",file=log)
log.close()
print_()
print_(file=log)
aLog.flush()
csv.write()
aLog.close()
def clean(self, region):
rmtree(self.master.name + "." + region, ignore_errors=True)
def prepare(self,
sol,
cName=None,
overrideParameters=None,
numberOfProcessors=None):
"""Do the actual preparing
:param numberOfProcessors: If set this overrides the value set in the
command line"""
if cName == None:
cName = sol.name
if self.opts.onlyVariables:
self.opts.verbose = True
vals = {}
vals, self.metaData = self.getDefaultValues(cName)
vals.update(
self.addDictValues(
"System", "Automatically defined values", {
"casePath":
'"' + path.abspath(cName) + '"',
"caseName":
'"' + path.basename(path.abspath(cName)) + '"',
"foamVersion":
foamVersion(),
"foamFork":
foamFork(),
"numberOfProcessors":
numberOfProcessors if numberOfProcessors != None else
self.opts.numberOfProcessors
}))
if len(self.opts.extensionAddition) > 0:
vals.update(
self.addDictValues(
"ExtensionAdditions",
"Additional extensions to be processed",
dict((e, True) for e in self.opts.extensionAddition)))
valsWithDefaults = set(vals.keys())
self.info("Looking for template values", cName)
for f in self.opts.valuesDicts:
self.info("Reading values from", f)
vals.update(
ParsedParameterFile(f, noHeader=True,
doMacroExpansion=True).getValueDict())
setValues = {}
for v in self.opts.values:
self.info("Updating values", v)
vals.update(eval(v))
setValues.update(eval(v))
if overrideParameters:
vals.update(overrideParameters)
unknownValues = set(vals.keys()) - valsWithDefaults
if len(unknownValues) > 0:
self.warning("Values for which no default was specified: " +
", ".join(unknownValues))
if self.opts.verbose and len(vals) > 0:
print_("\nUsed values\n")
nameLen = max(len("Name"), max(*[len(k) for k in vals.keys()]))
format = "%%%ds - %%s" % nameLen
print_(format % ("Name", "Value"))
print_("-" * 40)
for k, v in sorted(iteritems(vals)):
print_(format % (k, v))
print_("")
else:
self.info("\nNo values specified\n")
self.checkCorrectOptions(vals)
derivedScript = path.join(cName, self.opts.derivedParametersScript)
derivedAdded = None
derivedChanged = None
if path.exists(derivedScript):
self.info("Deriving variables in script", derivedScript)
scriptText = open(derivedScript).read()
glob = {}
oldVals = vals.copy()
exec_(scriptText, glob, vals)
derivedAdded = []
derivedChanged = []
for k, v in iteritems(vals):
if k not in oldVals:
derivedAdded.append(k)
elif vals[k] != oldVals[k]:
derivedChanged.append(k)
if len(derivedChanged) > 0 and (
not self.opts.allowDerivedChanges
and not configuration().getboolean("PrepareCase",
"AllowDerivedChanges")):
self.error(
self.opts.derivedParametersScript, "changed values of",
" ".join(derivedChanged),
"\nTo allow this set --allow-derived-changes or the configuration item 'AllowDerivedChanges'"
)
if len(derivedAdded) > 0:
self.info("Added values:", " ".join(derivedAdded))
if len(derivedChanged) > 0:
self.info("Changed values:", " ".join(derivedChanged))
if len(derivedAdded) == 0 and len(derivedChanged) == 0:
self.info("Nothing added or changed")
else:
self.info("No script", derivedScript, "for derived values")
if self.opts.onlyVariables:
return
self.__writeToStateFile(sol, "Starting")
if self.opts.doClear:
self.info("Clearing", cName)
self.__writeToStateFile(sol, "Clearing")
sol.clear(processor=True,
pyfoam=True,
vtk=True,
removeAnalyzed=True,
keepParallel=False,
clearHistory=False,
clearParameters=True,
additional=["postProcessing"])
self.__writeToStateFile(sol, "Done clearing")
if self.opts.writeParameters:
fName = path.join(cName, self.parameterOutFile)
self.info("Writing parameters to", fName)
with WriteParameterFile(fName, noHeader=True) as w:
w.content.update(vals, toString=True)
w["foamVersion"] = vals["foamVersion"]
w.writeFile()
if self.opts.writeReport:
fName = path.join(cName, self.parameterOutFile + ".rst")
self.info("Writing report to", fName)
with open(fName, "w") as w:
helper = RestructuredTextHelper(defaultHeading=1)
w.write(".. title:: " + self.__strip(vals["caseName"]) + "\n")
w.write(".. sectnum::\n")
w.write(".. header:: " + self.__strip(vals["caseName"]) + "\n")
w.write(".. header:: " + time.asctime() + "\n")
w.write(".. footer:: ###Page### / ###Total###\n\n")
w.write("Parameters set in case directory " +
helper.literal(self.__strip(vals["casePath"])) +
" at " + helper.emphasis(time.asctime()) + "\n\n")
w.write(".. contents::\n\n")
if len(self.opts.valuesDicts):
w.write(helper.heading("Parameter files"))
w.write("Parameters read from files\n\n")
w.write(
helper.enumerateList([
helper.literal(f) for f in self.opts.valuesDicts
]))
w.write("\n")
if len(setValues) > 0:
w.write(helper.heading("Overwritten parameters"))
w.write(
"These parameters were set from the command line\n\n")
w.write(helper.definitionList(setValues))
w.write("\n")
w.write(helper.heading("Parameters with defaults"))
w.write(self.makeReport(vals))
if len(unknownValues) > 0:
w.write(helper.heading("Unspecified parameters"))
w.write(
"If these parameters are actually used then specify them in "
+ helper.literal(self.defaultParameterFile) + "\n\n")
tab = helper.table(True)
for u in unknownValues:
tab.addRow(u)
tab.addItem("Value", vals[u])
w.write(str(tab))
if not derivedAdded is None:
w.write(helper.heading("Derived Variables"))
w.write("Script with derived Parameters" +
helper.literal(derivedScript) + "\n\n")
if len(derivedAdded) > 0:
w.write("These values were added:\n")
tab = helper.table(True)
for a in derivedAdded:
tab.addRow(a)
tab.addItem("Value", str(vals[a]))
w.write(str(tab))
if len(derivedChanged) > 0:
w.write("These values were changed:\n")
tab = helper.table(True)
for a in derivedChanged:
tab.addRow(a)
tab.addItem("Value", str(vals[a]))
tab.addItem("Old", str(oldVals[a]))
w.write(str(tab))
w.write("The code of the script:\n")
w.write(helper.code(scriptText))
self.addToCaseLog(cName)
for over in self.opts.overloadDirs:
self.info("Overloading files from", over)
self.__writeToStateFile(sol, "Overloading")
self.overloadDir(sol.name, over)
self.__writeToStateFile(sol, "Initial")
zeroOrig = path.join(sol.name, "0.org")
hasOrig = path.exists(zeroOrig)
cleanZero = True
if not hasOrig:
self.info("Not going to clean '0'")
self.opts.cleanDirectories.remove("0")
cleanZero = False
if self.opts.doCopy:
if hasOrig:
self.info("Found 0.org. Clearing 0")
zeroDir = path.join(sol.name, "0")
if path.exists(zeroDir):
rmtree(zeroDir)
else:
self.info("No 0-directory")
self.info("")
else:
cleanZero = False
if self.opts.doTemplates:
self.__writeToStateFile(sol, "Templates")
self.searchAndReplaceTemplates(
sol.name,
vals,
self.opts.templateExt,
ignoreDirectories=self.opts.ignoreDirectories)
self.info("")
backupZeroDir = None
if self.opts.doMeshCreate:
self.__writeToStateFile(sol, "Meshing")
if self.opts.meshCreateScript:
scriptName = path.join(sol.name, self.opts.meshCreateScript)
if not path.exists(scriptName):
self.error("Script", scriptName, "does not exist")
elif path.exists(path.join(sol.name, self.defaultMeshCreate)):
scriptName = path.join(sol.name, self.defaultMeshCreate)
else:
scriptName = None
if scriptName:
self.info("Executing", scriptName, "for mesh creation")
if self.opts.verbose:
echo = "Mesh: "
else:
echo = None
self.executeScript(scriptName, workdir=sol.name, echo=echo)
else:
self.info(
"No script for mesh creation found. Looking for 'blockMeshDict'"
)
if sol.blockMesh() != "":
self.info(sol.blockMesh(), "found. Executing 'blockMesh'")
bm = BasicRunner(argv=["blockMesh", "-case", sol.name])
bm.start()
if not bm.runOK():
self.error("Problem with blockMesh")
for r in sol.regions():
self.info("Checking region", r)
s = SolutionDirectory(sol.name,
region=r,
archive=None,
paraviewLink=False)
if s.blockMesh() != "":
self.info(s.blockMesh(),
"found. Executing 'blockMesh'")
bm = BasicRunner(argv=[
"blockMesh", "-case", sol.name, "-region", r
])
bm.start()
if not bm.runOK():
self.error("Problem with blockMesh")
self.info("")
if cleanZero and path.exists(zeroDir):
self.warning("Mesh creation recreated 0-directory")
if self.opts.keepZeroDirectoryFromMesh:
backupZeroDir = zeroDir + ".bakByPyFoam"
self.info("Backing up", zeroDir, "to", backupZeroDir)
move(zeroDir, backupZeroDir)
else:
self.info("Data in", zeroDir, "will be removed")
self.__writeToStateFile(sol, "Done Meshing")
if self.opts.doCopy:
self.__writeToStateFile(sol, "Copying")
self.copyOriginals(sol.name)
self.info("")
if backupZeroDir:
self.info("Copying backups from", backupZeroDir, "to", zeroDir)
self.overloadDir(zeroDir, backupZeroDir)
self.info("Removing backup", backupZeroDir)
rmtree(backupZeroDir)
if self.opts.doPostTemplates:
self.__writeToStateFile(sol, "Post-templates")
self.searchAndReplaceTemplates(
sol.name,
vals,
self.opts.postTemplateExt,
ignoreDirectories=self.opts.ignoreDirectories)
self.info("")
if self.opts.doCaseSetup:
self.__writeToStateFile(sol, "Case setup")
if self.opts.caseSetupScript:
scriptName = path.join(sol.name, self.opts.caseSetupScript)
if not path.exists(scriptName):
self.error("Script", scriptName, "does not exist")
elif path.exists(path.join(sol.name, self.defaultCaseSetup)):
scriptName = path.join(sol.name, self.defaultCaseSetup)
else:
scriptName = None
if scriptName:
self.info("Executing", scriptName, "for case setup")
if self.opts.verbose:
echo = "Case:"
else:
echo = None
self.executeScript(scriptName, workdir=sol.name, echo=echo)
elif path.exists(path.join(sol.name, "system", "setFieldsDict")):
self.info(
"So setup script found. But 'setFieldsDict'. Executing setFields"
)
sf = BasicRunner(argv=["setFields", "-case", sol.name])
sf.start()
if not sf.runOK():
self.error("Problem with setFields")
else:
self.info("No script for case-setup found. Nothing done")
self.info("")
self.__writeToStateFile(sol, "Done case setup")
if self.opts.doFinalTemplates:
self.__writeToStateFile(sol, "Final templates")
self.searchAndReplaceTemplates(
sol.name,
vals,
self.opts.finalTemplateExt,
ignoreDirectories=self.opts.ignoreDirectories)
if self.opts.doTemplateClean:
self.info("Clearing templates")
for d in self.opts.cleanDirectories:
for e in [
self.opts.templateExt, self.opts.postTemplateExt,
self.opts.finalTemplateExt
]:
self.cleanExtension(path.join(sol.name, d), e)
self.info("")
self.info("Case setup finished")
self.__writeToStateFile(sol, "Finished OK")
def prepare(self, sol, cName=None, overrideParameters=None):
if cName == None:
cName = sol.name
if self.opts.onlyVariables:
self.opts.verbose = True
vals = {}
vals["casePath"] = '"' + path.abspath(cName) + '"'
vals["caseName"] = '"' + path.basename(path.abspath(cName)) + '"'
vals["foamVersion"] = foamVersion()
vals["foamFork"] = foamFork()
if self.opts.verbose:
print_("Looking for template values", cName)
for f in self.opts.valuesDicts:
if self.opts.verbose:
print_("Reading values from", f)
vals.update(
ParsedParameterFile(f, noHeader=True,
doMacroExpansion=True).getValueDict())
for v in self.opts.values:
if self.opts.verbose:
print_("Updating values", v)
vals.update(eval(v))
if overrideParameters:
vals.update(overrideParameters)
if self.opts.verbose and len(vals) > 0:
print_("\nUsed values\n")
nameLen = max(len("Name"), max(*[len(k) for k in vals.keys()]))
format = "%%%ds - %%s" % nameLen
print_(format % ("Name", "Value"))
print_("-" * 40)
for k, v in sorted(iteritems(vals)):
print_(format % (k, v))
print_("")
elif self.opts.verbose:
print_("\nNo values specified\n")
if self.opts.onlyVariables:
return
if self.opts.doClear:
if self.opts.verbose:
print_("Clearing", cName)
sol.clear(processor=True,
pyfoam=True,
vtk=True,
removeAnalyzed=True,
keepParallel=False,
clearHistory=False,
clearParameters=True,
additional=["postProcessing"])
if self.opts.writeParameters:
fName = path.join(cName, self.parameterOutFile)
if self.opts.verbose:
print_("Writing parameters to", fName)
with WriteParameterFile(fName, noHeader=True) as w:
w.content.update(vals, toString=True)
w["foamVersion"] = vals["foamVersion"]
w.writeFile()
self.addToCaseLog(cName)
for over in self.opts.overloadDirs:
if self.opts.verbose:
print_("Overloading files from", over)
self.overloadDir(sol.name, over)
zeroOrig = path.join(sol.name, "0.org")
hasOrig = path.exists(zeroOrig)
if not hasOrig:
if self.opts.verbose:
print_("Not going to clean '0'")
self.opts.cleanDirectories.remove("0")
if self.opts.doCopy:
if hasOrig:
if self.opts.verbose:
print_("Found 0.org. Clearing 0")
zeroDir = path.join(sol.name, "0")
if path.exists(zeroDir):
rmtree(zeroDir)
elif self.opts.verbose:
print_("No 0-directory")
if self.opts.verbose:
print_("")
if self.opts.doTemplates:
self.searchAndReplaceTemplates(sol.name, vals,
self.opts.templateExt)
if self.opts.verbose:
print_("")
if self.opts.doMeshCreate:
if self.opts.meshCreateScript:
scriptName = path.join(sol.name, self.opts.meshCreateScript)
if not path.exists(scriptName):
self.error("Script", scriptName, "does not exist")
elif path.exists(path.join(sol.name, self.defaultMeshCreate)):
scriptName = path.join(sol.name, self.defaultMeshCreate)
else:
scriptName = None
if scriptName:
if self.opts.verbose:
print_("Executing", scriptName, "for mesh creation")
if self.opts.verbose:
echo = "Mesh: "
else:
echo = None
result = "".join(
execute([scriptName], workdir=sol.name, echo=echo))
open(scriptName + ".log", "w").write(result)
else:
if self.opts.verbose:
print_(
"No script for mesh creation found. Looking for 'blockMeshDict'"
)
if sol.blockMesh() != "":
if self.opts.verbose:
print_(sol.blockMesh(), "found. Executing 'blockMesh'")
bm = BasicRunner(argv=["blockMesh", "-case", sol.name])
bm.start()
if not bm.runOK():
self.error("Problem with blockMesh")
if self.opts.verbose:
print_("")
if self.opts.doCopy:
self.copyOriginals(sol.name)
if self.opts.verbose:
print_("")
if self.opts.doPostTemplates:
self.searchAndReplaceTemplates(sol.name, vals,
self.opts.postTemplateExt)
if self.opts.verbose:
print_("")
if self.opts.doCaseSetup:
if self.opts.caseSetupScript:
scriptName = path.join(sol.name, self.opts.caseSetupScript)
if not path.exists(scriptName):
self.error("Script", scriptName, "does not exist")
elif path.exists(path.join(sol.name, self.defaultCaseSetup)):
scriptName = path.join(sol.name, self.defaultCaseSetup)
else:
scriptName = None
if scriptName:
if self.opts.verbose:
print_("Executing", scriptName, "for case setup")
if self.opts.verbose:
echo = "Case:"
else:
echo = None
result = "".join(
execute([scriptName], workdir=sol.name, echo=echo))
open(scriptName + ".log", "w").write(result)
else:
if self.opts.verbose:
print_("No script for case-setup found. Nothing done")
if self.opts.verbose:
print_("")
if self.opts.doTemplateClean:
if self.opts.verbose:
print_("Clearing templates")
for d in self.opts.cleanDirectories:
for e in [self.opts.templateExt, self.opts.postTemplateExt]:
self.cleanExtension(path.join(sol.name, d), e)
if self.opts.verbose:
print_("")
if self.opts.verbose:
print_("Case setup finished")
def prepare(self,sol,
cName=None,
overrideParameters=None,
numberOfProcessors=None):
"""Do the actual preparing
@param numberOfProcessors: If set this overrides the value set in the
command line"""
if cName==None:
cName=sol.name
if self.opts.onlyVariables:
self.opts.verbose=True
vals={}
vals,self.metaData=self.getDefaultValues(cName)
vals.update(self.addDictValues("System",
"Automatically defined values",
{
"casePath" : '"'+path.abspath(cName)+'"',
"caseName" : '"'+path.basename(path.abspath(cName))+'"',
"foamVersion" : foamVersion(),
"foamFork" : foamFork(),
"numberOfProcessors" : numberOfProcessors if numberOfProcessors!=None else self.opts.numberOfProcessors
}))
if len(self.opts.extensionAddition)>0:
vals.update(self.addDictValues("ExtensionAdditions",
"Additional extensions to be processed",
dict((e,True) for e in self.opts.extensionAddition)))
valsWithDefaults=set(vals.keys())
self.info("Looking for template values",cName)
for f in self.opts.valuesDicts:
self.info("Reading values from",f)
vals.update(ParsedParameterFile(f,
noHeader=True,
doMacroExpansion=True).getValueDict())
setValues={}
for v in self.opts.values:
self.info("Updating values",v)
vals.update(eval(v))
setValues.update(eval(v))
if overrideParameters:
vals.update(overrideParameters)
unknownValues=set(vals.keys())-valsWithDefaults
if len(unknownValues)>0:
self.warning("Values for which no default was specified: "+
", ".join(unknownValues))
if self.opts.verbose and len(vals)>0:
print_("\nUsed values\n")
nameLen=max(len("Name"),
max(*[len(k) for k in vals.keys()]))
format="%%%ds - %%s" % nameLen
print_(format % ("Name","Value"))
print_("-"*40)
for k,v in sorted(iteritems(vals)):
print_(format % (k,v))
print_("")
else:
self.info("\nNo values specified\n")
self.checkCorrectOptions(vals)
derivedScript=path.join(cName,self.opts.derivedParametersScript)
if path.exists(derivedScript):
self.info("Deriving variables in script",derivedScript)
scriptText=open(derivedScript).read()
glob={}
oldVals=vals.copy()
exec_(scriptText,glob,vals)
added=[]
changed=[]
for k,v in iteritems(vals):
if k not in oldVals:
added.append(k)
elif vals[k]!=oldVals[k]:
changed.append(k)
if len(changed)>0 and (not self.opts.allowDerivedChanges and not configuration().getboolean("PrepareCase","AllowDerivedChanges")):
self.error(self.opts.derivedParametersScript,
"changed values of"," ".join(changed),
"\nTo allow this set --allow-derived-changes or the configuration item 'AllowDerivedChanges'")
if len(added)>0:
self.info("Added values:"," ".join(added))
if len(changed)>0:
self.info("Changed values:"," ".join(changed))
if len(added)==0 and len(changed)==0:
self.info("Nothing added or changed")
else:
self.info("No script",derivedScript,"for derived values")
if self.opts.onlyVariables:
return
if self.opts.doClear:
self.info("Clearing",cName)
sol.clear(processor=True,
pyfoam=True,
vtk=True,
removeAnalyzed=True,
keepParallel=False,
clearHistory=False,
clearParameters=True,
additional=["postProcessing"])
if self.opts.writeParameters:
fName=path.join(cName,self.parameterOutFile)
self.info("Writing parameters to",fName)
with WriteParameterFile(fName,noHeader=True) as w:
w.content.update(vals,toString=True)
w["foamVersion"]=vals["foamVersion"]
w.writeFile()
if self.opts.writeReport:
fName=path.join(cName,self.parameterOutFile+".rst")
self.info("Writing report to",fName)
with open(fName,"w") as w:
helper=RestructuredTextHelper(defaultHeading=1)
w.write(".. title:: "+self.__strip(vals["caseName"])+"\n")
w.write(".. sectnum::\n")
w.write(".. header:: "+self.__strip(vals["caseName"])+"\n")
w.write(".. header:: "+time.asctime()+"\n")
w.write(".. footer:: ###Page### / ###Total###\n\n")
w.write("Parameters set in case directory "+
helper.literal(self.__strip(vals["casePath"]))+" at "+
helper.emphasis(time.asctime())+"\n\n")
w.write(".. contents::\n\n")
if len(self.opts.valuesDicts):
w.write(helper.heading("Parameter files"))
w.write("Parameters read from files\n\n")
w.write(helper.enumerateList([helper.literal(f) for f in self.opts.valuesDicts]))
w.write("\n")
if len(setValues)>0:
w.write(helper.heading("Overwritten parameters"))
w.write("These parameters were set from the command line\n\n")
w.write(helper.definitionList(setValues))
w.write("\n")
w.write(helper.heading("Parameters with defaults"))
w.write(self.makeReport(vals))
if len(unknownValues)>0:
w.write(helper.heading("Unspecified parameters"))
w.write("If these parameters are actually used then specify them in "+
helper.literal(self.defaultParameterFile)+"\n\n")
tab=helper.table(True)
for u in unknownValues:
tab.addRow(u)
tab.addItem("Value",vals[u])
w.write(str(tab))
self.addToCaseLog(cName)
for over in self.opts.overloadDirs:
self.info("Overloading files from",over)
self.overloadDir(sol.name,over)
zeroOrig=path.join(sol.name,"0.org")
hasOrig=path.exists(zeroOrig)
cleanZero=True
if not hasOrig:
self.info("Not going to clean '0'")
self.opts.cleanDirectories.remove("0")
cleanZero=False
if self.opts.doCopy:
if hasOrig:
self.info("Found 0.org. Clearing 0")
zeroDir=path.join(sol.name,"0")
if path.exists(zeroDir):
rmtree(zeroDir)
else:
self.info("No 0-directory")
self.info("")
else:
cleanZero=False
if self.opts.doTemplates:
self.searchAndReplaceTemplates(sol.name,
vals,
self.opts.templateExt)
self.info("")
backupZeroDir=None
if self.opts.doMeshCreate:
if self.opts.meshCreateScript:
scriptName=path.join(sol.name,self.opts.meshCreateScript)
if not path.exists(scriptName):
self.error("Script",scriptName,"does not exist")
elif path.exists(path.join(sol.name,self.defaultMeshCreate)):
scriptName=path.join(sol.name,self.defaultMeshCreate)
else:
scriptName=None
if scriptName:
self.info("Executing",scriptName,"for mesh creation")
if self.opts.verbose:
echo="Mesh: "
else:
echo=None
result="".join(execute([scriptName],workdir=sol.name,echo=echo))
open(scriptName+".log","w").write(result)
else:
self.info("No script for mesh creation found. Looking for 'blockMeshDict'")
if sol.blockMesh()!="":
self.info(sol.blockMesh(),"found. Executing 'blockMesh'")
bm=BasicRunner(argv=["blockMesh","-case",sol.name])
bm.start()
if not bm.runOK():
self.error("Problem with blockMesh")
for r in sol.regions():
self.info("Checking region",r)
s=SolutionDirectory(sol.name,region=r,
archive=None,paraviewLink=False)
if s.blockMesh()!="":
self.info(s.blockMesh(),"found. Executing 'blockMesh'")
bm=BasicRunner(argv=["blockMesh","-case",sol.name,
"-region",r])
bm.start()
if not bm.runOK():
self.error("Problem with blockMesh")
self.info("")
if cleanZero and path.exists(zeroDir):
self.warning("Mesh creation recreated 0-directory")
if self.opts.keepZeroDirectoryFromMesh:
backupZeroDir=zeroDir+".bakByPyFoam"
self.info("Backing up",zeroDir,"to",backupZeroDir)
move(zeroDir,backupZeroDir)
else:
self.info("Data in",zeroDir,"will be removed")
if self.opts.doCopy:
self.copyOriginals(sol.name)
self.info("")
if backupZeroDir:
self.info("Copying backups from",backupZeroDir,"to",zeroDir)
self.overloadDir(zeroDir,backupZeroDir)
self.info("Removing backup",backupZeroDir)
rmtree(backupZeroDir)
if self.opts.doPostTemplates:
self.searchAndReplaceTemplates(sol.name,
vals,
self.opts.postTemplateExt)
self.info("")
if self.opts.doCaseSetup:
if self.opts.caseSetupScript:
scriptName=path.join(sol.name,self.opts.caseSetupScript)
if not path.exists(scriptName):
self.error("Script",scriptName,"does not exist")
elif path.exists(path.join(sol.name,self.defaultCaseSetup)):
scriptName=path.join(sol.name,self.defaultCaseSetup)
else:
scriptName=None
if scriptName:
self.info("Executing",scriptName,"for case setup")
if self.opts.verbose:
echo="Case:"
else:
echo=None
result="".join(execute([scriptName],workdir=sol.name,echo=echo))
open(scriptName+".log","w").write(result)
else:
self.info("No script for case-setup found. Nothing done")
self.info("")
if self.opts.doFinalTemplates:
self.searchAndReplaceTemplates(sol.name,
vals,
self.opts.finalTemplateExt)
if self.opts.doTemplateClean:
self.info("Clearing templates")
for d in self.opts.cleanDirectories:
for e in [self.opts.templateExt,
self.opts.postTemplateExt,
self.opts.finalTemplateExt]:
self.cleanExtension(path.join(sol.name,d),e)
self.info("")
self.info("Case setup finished")
def run(self):
config = ConfigParser.ConfigParser()
files = self.parser.getArgs()
good = config.read(files)
# will work with 2.4
# if len(good)!=len(files):
# print_("Problem while trying to parse files",files)
# print_("Only ",good," could be parsed")
# sys.exit(-1)
benchName = config.get("General", "name")
if self.opts.nameAddition != None:
benchName += "_" + self.opts.nameAddition
if self.opts.foamVersion != None:
benchName += "_v" + self.opts.foamVersion
isParallel = config.getboolean("General", "parallel")
lam = None
if isParallel:
nrCpus = config.getint("General", "nProcs")
machineFile = config.get("General", "machines")
if not path.exists(machineFile):
self.error("Machine file ", machineFile,
"needed for parallel run")
lam = LAMMachine(machineFile, nr=nrCpus)
if lam.cpuNr() > nrCpus:
self.error("Wrong number of CPUs: ", lam.cpuNr())
print_("Running parallel on", lam.cpuNr(), "CPUs")
if config.has_option("General", "casesDirectory"):
casesDirectory = path.expanduser(
config.get("General", "casesDirectory"))
else:
casesDirectory = foamTutorials()
if not path.exists(casesDirectory):
self.error("Directory", casesDirectory,
"needed with the benchmark cases is missing")
else:
print_("Using cases from directory", casesDirectory)
benchCases = []
config.remove_section("General")
for sec in config.sections():
print_("Reading: ", sec)
skipIt = False
skipReason = ""
if config.has_option(sec, "skip"):
skipIt = config.getboolean(sec, "skip")
skipReason = "Switched off in file"
if self.opts.excases != None and not skipIt:
for p in self.opts.excases:
if fnmatch(sec, p):
skipIt = True
skipReason = "Switched off by pattern '" + p + "'"
if self.opts.cases != None:
for p in self.opts.cases:
if fnmatch(sec, p):
skipIt = False
skipReason = ""
if skipIt:
print_("Skipping case ..... Reason:" + skipReason)
continue
sol = config.get(sec, "solver")
cas = config.get(sec, "case")
pre = eval(config.get(sec, "prepare"))
preCon = []
if config.has_option(sec, "preControlDict"):
preCon = eval(config.get(sec, "preControlDict"))
con = eval(config.get(sec, "controlDict"))
bas = config.getfloat(sec, "baseline")
wei = config.getfloat(sec, "weight")
add = []
if config.has_option(sec, "additional"):
add = eval(config.get(sec, "additional"))
print_("Adding: ", add)
util = []
if config.has_option(sec, "utilities"):
util = eval(config.get(sec, "utilities"))
print_("Utilities: ", util)
nr = 99999
if config.has_option(sec, "nr"):
nr = eval(config.get(sec, "nr"))
sp = None
if config.has_option(sec, "blockSplit"):
sp = eval(config.get(sec, "blockSplit"))
toRm = []
if config.has_option(sec, "filesToRemove"):
toRm = eval(config.get(sec, "filesToRemove"))
setInit = []
if config.has_option(sec, "setInitial"):
setInit = eval(config.get(sec, "setInitial"))
parallelOK = False
if config.has_option(sec, "parallelOK"):
parallelOK = config.getboolean(sec, "parallelOK")
deMet = ["metis"]
if config.has_option(sec, "decomposition"):
deMet = config.get(sec, "decomposition").split()
if deMet[0] == "metis":
pass
elif deMet[0] == "simple":
if len(deMet) < 2:
deMet.append(0)
else:
deMet[1] = int(deMet[1])
else:
print_("Unimplemented decomposition method", deMet[0],
"switching to metis")
deMet = ["metis"]
if isParallel == False or parallelOK == True:
if path.exists(path.join(casesDirectory, sol, cas)):
benchCases.append(
(nr, sec, sol, cas, pre, con, preCon, bas, wei, add,
util, sp, toRm, setInit, deMet))
else:
print_("Skipping", sec, "because directory",
path.join(casesDirectory, sol, cas),
"could not be found")
else:
print_("Skipping", sec, "because not parallel")
benchCases.sort()
parallelString = ""
if isParallel:
parallelString = ".cpus=" + str(nrCpus)
resultFile = open(
"Benchmark." + benchName + "." + uname()[1] + parallelString +
".results", "w")
totalSpeedup = 0
minSpeedup = None
maxSpeedup = None
totalWeight = 0
runsOK = 0
currentEstimate = 1.
print_("\nStart Benching\n")
csv = CSVCollection("Benchmark." + benchName + "." + uname()[1] +
parallelString + ".csv")
# csvHeaders=["description","solver","case","caseDir","base",
# "benchmark","machine","arch","cpus","os","version",
# "wallclocktime","cputime","cputimeuser","cputimesystem","maxmemory","cpuusage","speedup"]
for nr, description, solver, case, prepare, control, preControl, base, weight, additional, utilities, split, toRemove, setInit, decomposition in benchCases:
# control.append( ("endTime",-2000) )
print_("Running Benchmark: ", description)
print_("Solver: ", solver)
print_("Case: ", case)
caseName = solver + "_" + case + "_" + benchName + "." + uname(
)[1] + ".case"
print_("Short name: ", caseName)
caseDir = caseName + ".runDir"
csv["description"] = description
csv["solver"] = solver
csv["case"] = case
csv["caseDir"] = caseDir
csv["base"] = base
csv["benchmark"] = benchName
csv["machine"] = uname()[1]
csv["arch"] = uname()[4]
if lam == None:
csv["cpus"] = 1
else:
csv["cpus"] = lam.cpuNr()
csv["os"] = uname()[0]
csv["version"] = uname()[2]
workDir = path.realpath(path.curdir)
orig = SolutionDirectory(path.join(casesDirectory, solver, case),
archive=None,
paraviewLink=False)
for a in additional + utilities:
orig.addToClone(a)
orig.cloneCase(path.join(workDir, caseDir))
if oldApp():
argv = [solver, workDir, caseDir]
else:
argv = [solver, "-case", path.join(workDir, caseDir)]
run = BasicRunner(silent=True,
argv=argv,
logname="BenchRunning",
lam=lam)
runDir = run.getSolutionDirectory()
controlFile = ParameterFile(runDir.controlDict())
for name, value in preControl:
print_("Setting parameter", name, "to", value,
"in controlDict")
controlFile.replaceParameter(name, value)
for rm in toRemove:
fn = path.join(caseDir, rm)
print_("Removing file", fn)
remove(fn)
for field, bc, val in setInit:
print_("Setting", field, "on", bc, "to", val)
SolutionFile(runDir.initialDir(),
field).replaceBoundary(bc, val)
oldDeltaT = controlFile.replaceParameter("deltaT", 0)
for u in utilities:
print_("Building utility ", u)
execute("wmake 2>&1 >%s %s" % (path.join(
caseDir, "BenchCompile." + u), path.join(caseDir, u)))
print_("Preparing the case: ")
if lam != None:
prepare = prepare + [("decomposePar", "")]
if decomposition[0] == "metis":
lam.writeMetis(
SolutionDirectory(path.join(workDir, caseDir)))
elif decomposition[0] == "simple":
lam.writeSimple(
SolutionDirectory(path.join(workDir, caseDir)),
decomposition[1])
if split:
print_("Splitting the mesh:", split)
bm = BlockMesh(runDir.blockMesh())
bm.refineMesh(split)
for pre, post in prepare:
print_("Doing ", pre, " ....")
post = post.replace("%case%", caseDir)
if oldApp():
args = string.split("%s %s %s %s" %
(pre, workDir, caseDir, post))
else:
args = string.split(
"%s -case %s %s" %
(pre, path.join(workDir, caseDir), post))
util = BasicRunner(silent=True,
argv=args,
logname="BenchPrepare_" + pre)
util.start()
controlFile.replaceParameter("deltaT", oldDeltaT)
# control.append(("endTime",-1000))
for name, value in control:
print_("Setting parameter", name, "to", value,
"in controlDict")
controlFile.replaceParameter(name, value)
print_("Starting at ", asctime(localtime(time())))
print_(
" Baseline is %f, estimated speedup %f -> estimated end at %s "
% (base, currentEstimate,
asctime(localtime(time() + base / currentEstimate))))
print_("Running the case ....")
run.start()
speedup = None
cpuUsage = 0
speedupOut = -1
try:
speedup = base / run.run.wallTime()
cpuUsage = 100. * run.run.cpuTime() / run.run.wallTime()
except ZeroDivisionError:
print_("Division by Zero: ", run.run.wallTime())
if not run.runOK():
print_("\nWARNING!!!!")
print_(
"Run had a problem, not using the results. Check the log\n"
)
speedup = None
if speedup != None:
speedupOut = speedup
totalSpeedup += speedup * weight
totalWeight += weight
runsOK += 1
if maxSpeedup == None:
maxSpeedup = speedup
elif speedup > maxSpeedup:
maxSpeedup = speedup
if minSpeedup == None:
minSpeedup = speedup
elif speedup < minSpeedup:
minSpeedup = speedup
print_("Wall clock: ", run.run.wallTime())
print_("Speedup: ", speedup, " (Baseline: ", base, ")")
print_("CPU Time: ", run.run.cpuTime())
print_("CPU Time User: "******"CPU Time System: ", run.run.cpuSystemTime())
print_("Memory: ", run.run.usedMemory())
print_("CPU Usage: %6.2f%%" % (cpuUsage))
csv["wallclocktime"] = run.run.wallTime()
csv["cputime"] = run.run.cpuTime()
csv["cputimeuser"] = run.run.cpuUserTime()
csv["cputimesystem"] = run.run.cpuSystemTime()
csv["maxmemory"] = run.run.usedMemory()
csv["cpuusage"] = cpuUsage
if speedup != None:
csv["speedup"] = speedup
else:
csv["speedup"] = "##"
csv.write()
resultFile.write(
"Case %s WallTime %g CPUTime %g UserTime %g SystemTime %g Memory %g MB Speedup %g\n"
% (caseName, run.run.wallTime(), run.run.cpuTime(),
run.run.cpuUserTime(), run.run.cpuSystemTime(),
run.run.usedMemory(), speedupOut))
resultFile.flush()
if speedup != None:
currentEstimate = totalSpeedup / totalWeight
if self.opts.removeCases:
print_("Clearing case", end=" ")
if speedup == None:
print_("not ... because it failed")
else:
print_("completely")
rmtree(caseDir, ignore_errors=True)
print_()
print_()
