def lookForCases(d):
for n in tqdm(listdir(d),
unit="entries",
leave=False,
desc=path.basename(path.abspath(d)),
disable=not self.opts.progressBar):
if not self.fnmatch(n):
continue
cName=path.join(d,n)
if path.isdir(cName):
try:
sol=SolutionDirectory(cName,archive=None,paraviewLink=False)
if sol.isValid():
if self.opts.progress:
print_("Processing",cName)
data={}
data["mtime"]=stat(cName)[ST_MTIME]
times=sol.getTimes()
try:
data["first"]=times[0]
except IndexError:
data["first"]="None"
try:
data["last"]=times[-1]
except IndexError:
data["last"]="None"
data["nrSteps"]=len(times)
data["procs"]=sol.nrProcs()
data["pFirst"]=-1
data["pLast"]=-1
data["nrParallel"]=-1
if self.opts.parallel:
pTimes=sol.getParallelTimes()
data["nrParallel"]=len(pTimes)
if len(pTimes)>0:
data["pFirst"]=pTimes[0]
data["pLast"]=pTimes[-1]
data["name"]=cName
data["diskusage"]=-1
if self.opts.diskusage:
data["diskusage"]=diskUsage(cName)
totalDiskusage+=data["diskusage"]
if self.opts.parallel:
for f in listdir(cName):
if re.compile("processor[0-9]+").match(f):
data["mtime"]=max(stat(path.join(cName,f))[ST_MTIME],data["mtime"])
if self.opts.state or self.opts.estimateEndTime:
try:
data["startedAt"]=time.mktime(time.strptime(self.readState(sol,"StartedAt")))
except ValueError:
data["startedAt"]="nix"
if self.opts.state:
try:
data["nowTime"]=float(self.readState(sol,"CurrentTime"))
except ValueError:
data["nowTime"]=None
try:
data["lastOutput"]=time.mktime(time.strptime(self.readState(sol,"LastOutputSeen")))
except ValueError:
data["lastOutput"]="nix"
data["state"]=self.readState(sol,"TheState")
if data["state"]=="Running":
try:
gone=time.time()-data["lastOutput"]
if gone>self.opts.deadThreshold:
data["state"]="Dead "+humanReadableDuration(gone)
except KeyError:
pass
except TypeError:
pass
if self.opts.startEndTime or self.opts.estimateEndTime:
try:
ctrlDict=ParsedParameterFile(sol.controlDict(),doMacroExpansion=True)
except PyFoamParserError:
# Didn't work with Macro expansion. Let's try without
try:
ctrlDict=ParsedParameterFile(sol.controlDict())
except PyFoamParserError:
ctrlDict=None
if ctrlDict:
data["startTime"]=ctrlDict["startTime"]
data["endTime"]=ctrlDict["endTime"]
else:
data["startTime"]=None
data["endTime"]=None
if self.opts.estimateEndTime:
data["endTimeEstimate"]=None
if self.readState(sol,"TheState")=="Running":
gone=time.time()-data["startedAt"]
try:
current=float(self.readState(sol,"CurrentTime"))
frac=(current-data["startTime"])/(data["endTime"]-data["startTime"])
except ValueError:
frac=0
if frac>0:
data["endTimeEstimate"]=data["startedAt"]+gone/frac
if self.opts.hgInfo:
if path.isdir(path.join(cName,".hg")):
from stat import ST_ATIME
prevStat=stat(cName)
try:
data["hgInfo"]=sub.Popen(["hg", "id",
"-R",cName,
"-b","-n","-i"], stdout=sub.PIPE).communicate()[0].strip()
except OSError:
data["hgInfo"]="<hg not working>"
postStat=stat(cName)
if prevStat[ST_MTIME]!=postStat[ST_MTIME]:
# hg seems to modify the modification time of the directory. So reset it
os.utime(cName,(postStat[ST_ATIME],prevStat[ST_MTIME]))
else:
data["hgInfo"]="<no .hg directory>"
if len(customData)>0 or self.opts.hostname:
fn=None
pickleFile=None
if useSolverInData:
data["solver"]="none found"
# try to find the oldest pickled file
dirAndTime=[]
for f in ["pickledData","pickledUnfinishedData","pickledStartData"]:
for g in glob(path.join(cName,"*.analyzed")):
pName=path.join(g,f)
base=path.basename(g)
if base.find("PyFoamRunner.")==0:
solverName=base[len("PyFoamRunner."):-len(".analyzed")]
else:
solverName=None
if path.exists(pName):
dirAndTime.append((path.getmtime(pName),solverName,pName))
dirAndTime.sort(key=lambda x:x[0])
if len(dirAndTime)>0:
data["solver"]=dirAndTime[-1][1]
pickleFile=dirAndTime[-1][2]
solverName=data["solver"]
else:
solverName=self.opts.solverNameForCustom
if pickleFile:
fn=pickleFile
else:
for f in ["pickledData","pickledUnfinishedData","pickledStartData"]:
fp=path.join(cName,"PyFoamRunner."+solverName+".analyzed",f)
if path.exists(fp):
fn=fp
break
pickleOK=False
if fn:
try:
raw=pickle.Unpickler(open(fn,"rb")).load()
pickleOK=True
for n,spec in customData:
dt=raw
for k in spec:
try:
dt=dt[k]
except KeyError:
dt="No key '"+k+"'"
break
if isinstance(dt,string_types):
break
data[n]=dt
if self.opts.hostname:
try:
data["hostname"]=raw["hostname"].split(".")[0]
except KeyError:
data["hostname"]="<unspecified>"
except ValueError:
pass
if not pickleOK:
for n,spec in customData:
data[n]="<no file>"
if self.opts.hostname:
data["hostname"]="<no file>"
cData.append(data)
elif self.opts.recursive:
# print("Recurse",cName)
lookForCases(cName)
except OSError:
print_(cName,"is unreadable")
def lookForCases(d):
for n in tqdm(listdir(d),
unit="entries",
leave=False,
desc=path.basename(path.abspath(d)),
disable=not self.opts.progressBar):
if not self.fnmatch(n):
continue
cName = path.join(d, n)
if path.isdir(cName):
try:
sol = SolutionDirectory(cName,
archive=None,
paraviewLink=False)
if sol.isValid():
if self.opts.progress:
print_("Processing", cName)
data = {}
data["mtime"] = stat(cName)[ST_MTIME]
times = sol.getTimes()
try:
data["first"] = times[0]
except IndexError:
data["first"] = "None"
try:
data["last"] = times[-1]
except IndexError:
data["last"] = "None"
data["nrSteps"] = len(times)
data["procs"] = sol.nrProcs()
data["pFirst"] = -1
data["pLast"] = -1
data["nrParallel"] = -1
if self.opts.parallel:
pTimes = sol.getParallelTimes()
data["nrParallel"] = len(pTimes)
if len(pTimes) > 0:
data["pFirst"] = pTimes[0]
data["pLast"] = pTimes[-1]
data["name"] = cName
data["diskusage"] = -1
if self.opts.diskusage:
data["diskusage"] = diskUsage(cName)
totalDiskusage += data["diskusage"]
if self.opts.parallel:
for f in listdir(cName):
if re.compile("processor[0-9]+").match(f):
data["mtime"] = max(
stat(path.join(cName,
f))[ST_MTIME],
data["mtime"])
if self.opts.state or self.opts.estimateEndTime:
try:
data["startedAt"] = time.mktime(
time.strptime(
self.readState(sol, "StartedAt")))
except ValueError:
data["startedAt"] = "nix"
if self.opts.state:
try:
data["nowTime"] = float(
self.readState(sol, "CurrentTime"))
except ValueError:
data["nowTime"] = None
try:
data["lastOutput"] = time.mktime(
time.strptime(
self.readState(
sol, "LastOutputSeen")))
except ValueError:
data["lastOutput"] = "nix"
data["state"] = self.readState(sol, "TheState")
if data["state"] == "Running":
try:
gone = time.time() - data["lastOutput"]
if gone > self.opts.deadThreshold:
data[
"state"] = "Dead " + humanReadableDuration(
gone)
except KeyError:
pass
except TypeError:
pass
if self.opts.startEndTime or self.opts.estimateEndTime:
try:
ctrlDict = ParsedParameterFile(
sol.controlDict(),
doMacroExpansion=True)
except PyFoamParserError:
# Didn't work with Macro expansion. Let's try without
try:
ctrlDict = ParsedParameterFile(
sol.controlDict())
except PyFoamParserError:
ctrlDict = None
if ctrlDict:
data["startTime"] = ctrlDict["startTime"]
data["endTime"] = ctrlDict["endTime"]
else:
data["startTime"] = None
data["endTime"] = None
if self.opts.estimateEndTime:
data["endTimeEstimate"] = None
if self.readState(sol,
"TheState") == "Running":
gone = time.time() - data["startedAt"]
try:
current = float(
self.readState(sol, "CurrentTime"))
frac = (current - data["startTime"]
) / (data["endTime"] -
data["startTime"])
except ValueError:
frac = 0
if frac > 0:
data["endTimeEstimate"] = data[
"startedAt"] + gone / frac
if self.opts.hgInfo:
if path.isdir(path.join(cName, ".hg")):
from stat import ST_ATIME
prevStat = stat(cName)
try:
data["hgInfo"] = sub.Popen(
[
"hg", "id", "-R", cName, "-b",
"-n", "-i"
],
stdout=sub.PIPE).communicate(
)[0].strip()
except OSError:
data["hgInfo"] = "<hg not working>"
postStat = stat(cName)
if prevStat[ST_MTIME] != postStat[ST_MTIME]:
# hg seems to modify the modification time of the directory. So reset it
os.utime(cName, (postStat[ST_ATIME],
prevStat[ST_MTIME]))
else:
data["hgInfo"] = "<no .hg directory>"
if len(customData) > 0 or self.opts.hostname:
fn = None
pickleFile = None
if useSolverInData:
data["solver"] = "none found"
# try to find the oldest pickled file
dirAndTime = []
for f in [
"pickledData",
"pickledUnfinishedData",
"pickledStartData"
]:
for g in glob(
path.join(cName,
"*.analyzed")):
pName = path.join(g, f)
base = path.basename(g)
if base.find("PyFoamRunner.") == 0:
solverName = base[
len("PyFoamRunner."
):-len(".analyzed")]
else:
solverName = None
if path.exists(pName):
dirAndTime.append(
(path.getmtime(pName),
solverName, pName))
dirAndTime.sort(key=lambda x: x[0])
if len(dirAndTime) > 0:
data["solver"] = dirAndTime[-1][1]
pickleFile = dirAndTime[-1][2]
solverName = data["solver"]
else:
solverName = self.opts.solverNameForCustom
if pickleFile:
fn = pickleFile
else:
for f in [
"pickledData",
"pickledUnfinishedData",
"pickledStartData"
]:
fp = path.join(
cName, "PyFoamRunner." +
solverName + ".analyzed", f)
if path.exists(fp):
fn = fp
break
pickleOK = False
if fn:
try:
raw = pickle.Unpickler(open(
fn, "rb")).load()
pickleOK = True
for n, spec in customData:
dt = raw
for k in spec:
try:
dt = dt[k]
except KeyError:
dt = "No key '" + k + "'"
break
if isinstance(
dt, string_types):
break
data[n] = dt
if self.opts.hostname:
try:
data["hostname"] = raw[
"hostname"].split(".")[0]
except KeyError:
data[
"hostname"] = "<unspecified>"
except ValueError:
pass
if not pickleOK:
for n, spec in customData:
data[n] = "<no file>"
if self.opts.hostname:
data["hostname"] = "<no file>"
cData.append(data)
elif self.opts.recursive:
# print("Recurse",cName)
lookForCases(cName)
except OSError:
print_(cName, "is unreadable")
def doRegion(self, theRegion):
ReST = RestructuredTextHelper(defaultHeading=self.opts.headingLevel)
if self.opts.allRegions:
print_(
ReST.buildHeading("Region: ",
theRegion,
level=self.opts.headingLevel - 1))
sol = SolutionDirectory(self.parser.getArgs()[0],
archive=None,
parallel=self.opts.parallel,
paraviewLink=False,
region=theRegion)
if self.opts.all:
self.opts.caseSize = True
self.opts.shortBCreport = True
self.opts.longBCreport = True
self.opts.dimensions = True
self.opts.internal = True
self.opts.linearSolvers = True
self.opts.relaxationFactors = True
self.opts.processorMatrix = True
self.opts.decomposition = True
if self.opts.time:
try:
self.opts.time = sol.timeName(
sol.timeIndex(self.opts.time, minTime=True))
except IndexError:
error("The specified time", self.opts.time,
"doesn't exist in the case")
print_("Using time t=" + self.opts.time + "\n")
needsPolyBoundaries = False
needsInitialTime = False
if self.opts.longBCreport:
needsPolyBoundaries = True
needsInitialTime = True
if self.opts.shortBCreport:
needsPolyBoundaries = True
needsInitialTime = True
if self.opts.dimensions:
needsInitialTime = True
if self.opts.internal:
needsInitialTime = True
if self.opts.decomposition:
needsPolyBoundaries = True
defaultProc = None
if self.opts.parallel:
defaultProc = 0
if needsPolyBoundaries:
proc = None
boundary = BoundaryDict(
sol.name,
region=theRegion,
time=self.opts.time,
treatBinaryAsASCII=self.opts.boundaryTreatBinaryAsASCII,
processor=defaultProc)
boundMaxLen = 0
boundaryNames = []
for b in boundary:
if b.find("procBoundary") != 0:
boundaryNames.append(b)
if self.opts.patches != None:
tmp = boundaryNames
boundaryNames = []
for b in tmp:
for p in self.opts.patches:
if fnmatch(b, p):
boundaryNames.append(b)
break
if self.opts.expatches != None:
tmp = boundaryNames
boundaryNames = []
for b in tmp:
keep = True
for p in self.opts.expatches:
if fnmatch(b, p):
keep = False
break
if keep:
boundaryNames.append(b)
for b in boundaryNames:
boundMaxLen = max(boundMaxLen, len(b))
boundaryNames.sort()
if self.opts.time == None:
procTime = "constant"
else:
procTime = self.opts.time
if needsInitialTime:
fields = {}
if self.opts.time == None:
try:
time = sol.timeName(0)
except IndexError:
error("There is no timestep in the case")
else:
time = self.opts.time
tDir = sol[time]
nameMaxLen = 0
for f in tDir:
try:
fields[f.baseName()] = f.getContent(
listLengthUnparsed=self.opts.longlist,
treatBinaryAsASCII=self.opts.treatBinaryAsASCII,
doMacroExpansion=self.opts.doMacros)
nameMaxLen = max(nameMaxLen, len(f.baseName()))
except PyFoamParserError:
e = sys.exc_info()[
1] # Needed because python 2.5 does not support 'as e'
warning("Couldn't parse", f.name, "because of an error:",
e, " -> skipping")
fieldNames = list(fields.keys())
fieldNames.sort()
if self.opts.caseSize:
print_(ReST.heading("Size of the case"))
nFaces = 0
nPoints = 0
nCells = 0
if self.opts.parallel:
procs = list(range(sol.nrProcs()))
print_("Accumulated from", sol.nrProcs(), "processors")
else:
procs = [None]
for p in procs:
info = MeshInformation(sol.name,
processor=p,
region=theRegion,
time=self.opts.time)
nFaces += info.nrOfFaces()
nPoints += info.nrOfPoints()
try:
nCells += info.nrOfCells()
except:
nCells = "Not available"
tab = ReST.table()
tab[0] = ("Faces", nFaces)
tab[1] = ("Points", nPoints)
tab[2] = ("Cells", nCells)
print_(tab)
if self.opts.decomposition:
print_(ReST.heading("Decomposition"))
if sol.nrProcs() < 2:
print_("This case is not decomposed")
else:
print_("Case is decomposed for", sol.nrProcs(), "processors")
print_()
nCells = []
nFaces = []
nPoints = []
for p in sol.processorDirs():
info = MeshInformation(sol.name,
processor=p,
region=theRegion,
time=self.opts.time)
nPoints.append(info.nrOfPoints())
nFaces.append(info.nrOfFaces())
nCells.append(info.nrOfCells())
digits = int(
ceil(
log10(
max(sol.nrProcs(), max(nCells), max(nFaces),
max(nPoints))))) + 2
nameLen = max(len("Points"), boundMaxLen)
tab = ReST.table()
tab[0] = ["CPU"] + list(range(sol.nrProcs()))
tab.addLine()
tab[1] = ["Points"] + nPoints
tab[2] = ["Faces"] + nFaces
tab[3] = ["Cells"] + nCells
tab.addLine(head=True)
nr = 3
for b in boundaryNames:
nr += 1
tab[(nr, 0)] = b
for i, p in enumerate(sol.processorDirs()):
try:
nFaces = ParsedBoundaryDict(
sol.boundaryDict(processor=p,
region=theRegion,
time=self.opts.time),
treatBinaryAsASCII=self.opts.
boundaryTreatBinaryAsASCII)[b]["nFaces"]
except IOError:
nFaces = ParsedBoundaryDict(
sol.boundaryDict(processor=p,
region=theRegion),
treatBinaryAsASCII=self.opts.
boundaryTreatBinaryAsASCII)[b]["nFaces"]
except KeyError:
nFaces = 0
tab[(nr, i + 1)] = nFaces
print_(tab)
if self.opts.longBCreport:
print_(ReST.heading("The boundary conditions for t =", time))
for b in boundaryNames:
print_(
ReST.buildHeading("Boundary: ",
b,
level=self.opts.headingLevel + 1))
bound = boundary[b]
print_(":Type:\t", bound["type"])
if "physicalType" in bound:
print_(":Physical:\t", bound["physicalType"])
print_(":Faces:\t", bound["nFaces"])
print_()
heads = ["Field", "type"]
tab = ReST.table()
tab[0] = heads
tab.addLine(head=True)
for row, fName in enumerate(fieldNames):
tab[(row + 1, 0)] = fName
f = fields[fName]
if "boundaryField" not in f:
tab[(row + 1, 1)] = "Not a field file"
elif b not in f["boundaryField"]:
tab[(row + 1, 1)] = "MISSING !!!"
else:
bf = f["boundaryField"][b]
for k in bf:
try:
col = heads.index(k)
except ValueError:
col = len(heads)
tab[(0, col)] = k
heads.append(k)
cont = str(bf[k])
if type(bf[k]) == Field:
if bf[k].isBinary():
cont = bf[k].binaryString()
if cont.find("\n") >= 0:
tab[(row + 1,
col)] = cont[:cont.find("\n")] + "..."
else:
tab[(row + 1, col)] = cont
print_(tab)
if self.opts.shortBCreport:
print_(ReST.heading("Table of boundary conditions for t =", time))
types = {}
hasPhysical = False
for b in boundary:
if "physicalType" in boundary[b]:
hasPhysical = True
types[b] = {}
for fName in fields:
f = fields[fName]
try:
if b not in f["boundaryField"]:
types[b][fName] = "MISSING"
else:
types[b][fName] = f["boundaryField"][b]["type"]
except KeyError:
types[b][fName] = "Not a field"
tab = ReST.table()
tab[0] = [""] + boundaryNames
tab.addLine()
tab[(1, 0)] = "Patch Type"
for i, b in enumerate(boundaryNames):
tab[(1, i + 1)] = boundary[b]["type"]
nr = 2
if hasPhysical:
tab[(nr, 0)] = "Physical Type"
for i, b in enumerate(boundaryNames):
if "physicalType" in boundary[b]:
tab[(nr, i + 1)] = boundary[b]["physicalType"]
nr += 1
tab[(nr, 0)] = "Length"
for i, b in enumerate(boundaryNames):
tab[(nr, i + 1)] = boundary[b]["nFaces"]
nr += 1
tab.addLine(head=True)
for fName in fieldNames:
tab[(nr, 0)] = fName
for i, b in enumerate(boundaryNames):
tab[(nr, i + 1)] = types[b][fName]
nr += 1
print_(tab)
if self.opts.dimensions:
print_(ReST.heading("Dimensions of fields for t =", time))
tab = ReST.table()
tab[0] = ["Name"] + "[ kg m s K mol A cd ]".split()[1:-1]
tab.addLine(head=True)
for i, fName in enumerate(fieldNames):
f = fields[fName]
try:
dim = str(f["dimensions"]).split()[1:-1]
except KeyError:
dim = ["-"] * 7
tab[i + 1] = [fName] + dim
print_(tab)
if self.opts.internal:
print_(ReST.heading("Internal value of fields for t =", time))
tab = ReST.table()
tab[0] = ["Name", "Value"]
tab.addLine(head=True)
for i, fName in enumerate(fieldNames):
f = fields[fName]
try:
if f["internalField"].isBinary():
val = f["internalField"].binaryString()
else:
cont = str(f["internalField"])
if cont.find("\n") >= 0:
val = cont[:cont.find("\n")] + "..."
else:
val = cont
except KeyError:
val = "Not a field file"
tab[i + 1] = [fName, val]
print_(tab)
if self.opts.processorMatrix:
print_(ReST.heading("Processor matrix"))
if sol.nrProcs() < 2:
print_("This case is not decomposed")
else:
matrix = [[
0,
] * sol.nrProcs() for i in range(sol.nrProcs())]
for i, p in enumerate(sol.processorDirs()):
try:
bound = ParsedBoundaryDict(
sol.boundaryDict(processor=p,
region=theRegion,
time=self.opts.time),
treatBinaryAsASCII=self.opts.
boundaryTreatBinaryAsASCII)
except IOError:
bound = ParsedBoundaryDict(
sol.boundaryDict(processor=p,
treatBinaryAsASCII=self.opts.
boundaryTreatBinaryAsASCII,
region=theRegion),
treatBinaryAsASCII=self.opts.
boundaryTreatBinaryAsASCII)
for j in range(sol.nrProcs()):
name = "procBoundary%dto%d" % (j, i)
name2 = "procBoundary%dto%d" % (i, j)
if name in bound:
matrix[i][j] = bound[name]["nFaces"]
if name2 in bound:
matrix[i][j] = bound[name2]["nFaces"]
print_("Matrix of processor interactions (faces)")
print_()
tab = ReST.table()
tab[0] = ["CPU"] + list(range(sol.nrProcs()))
tab.addLine(head=True)
for i, col in enumerate(matrix):
tab[i + 1] = [i] + matrix[i]
print_(tab)
if self.opts.linearSolvers:
print_(ReST.heading("Linear Solvers"))
linTable = ReST.table()
fvSol = ParsedParameterFile(
path.join(sol.systemDir(), "fvSolution"),
treatBinaryAsASCII=self.opts.treatBinaryAsASCII)
allInfo = {}
for sName in fvSol["solvers"]:
raw = fvSol["solvers"][sName]
info = {}
if type(raw) in [dict, DictProxy]:
# fvSolution format in 1.7
try:
info["solver"] = raw["solver"]
except KeyError:
info["solver"] = "<none>"
solverData = raw
else:
info["solver"] = raw[0]
solverData = raw[1]
if type(solverData) in [dict, DictProxy]:
try:
info["tolerance"] = solverData["tolerance"]
except KeyError:
info["tolerance"] = 1.
try:
info["relTol"] = solverData["relTol"]
except KeyError:
info["relTol"] = 0.
else:
# the old (pre-1.5) fvSolution-format
info["tolerance"] = solverData
info["relTol"] = raw[2]
allInfo[sName] = info
linTable[0] = ["Name", "Solver", "Abs. Tolerance", "Relative Tol."]
linTable.addLine(head=True)
nr = 0
for n, i in iteritems(allInfo):
nr += 1
linTable[nr] = (n, i["solver"], i["tolerance"], i["relTol"])
print_(linTable)
if self.opts.relaxationFactors:
print_(ReST.heading("Relaxation"))
fvSol = ParsedParameterFile(
path.join(sol.systemDir(), "fvSolution"),
treatBinaryAsASCII=self.opts.treatBinaryAsASCII)
if "relaxationFactors" in fvSol:
relax = fvSol["relaxationFactors"]
tab = ReST.table()
tab[0] = ["Name", "Factor"]
tab.addLine(head=True)
nr = 0
if "fields" in relax or "equations" in relax:
# New syntax
for k in ["fields", "equations"]:
if k in relax:
for n, f in iteritems(relax[k]):
nr += 1
tab[nr] = [k + ": " + n, f]
else:
for n, f in iteritems(relax):
nr += 1
tab[nr] = [n, f]
print_(tab)
else:
print_("No relaxation factors defined for this case")
def run(self):
dirs=self.parser.getArgs()
if len(dirs)==0:
dirs=[path.curdir]
cData=[]
totalDiskusage=0
useSolverInData=False
self.hasState=False
customData=[]
for i,c in enumerate(self.opts.customData):
lst=c.split("=")
if len(lst)==2:
name,spec=lst
name+="_" # Make sure that there is no collision with standard-names
elif len(lst)==1:
name,spec="Custom%d" % (i+1),c
else:
self.error("Custom specification",c,"does not fit the pattern 'name=subs1::subs2::..'")
customData.append((name,spec.split("::")))
if len(customData)>0 and not self.opts.solverNameForCustom:
self.warning("Parameter '--solver-name-for-custom-data' should be set if '--custom-data' is used")
useSolverInData=True
for d in dirs:
for n in listdir(d):
cName=path.join(d,n)
if path.isdir(cName):
try:
sol=SolutionDirectory(cName,archive=None,paraviewLink=False)
if sol.isValid():
if self.opts.progress:
print_("Processing",cName)
data={}
data["mtime"]=stat(cName)[ST_MTIME]
times=sol.getTimes()
try:
data["first"]=times[0]
except IndexError:
data["first"]="None"
try:
data["last"]=times[-1]
except IndexError:
data["last"]="None"
data["nrSteps"]=len(times)
data["procs"]=sol.nrProcs()
data["pFirst"]=-1
data["pLast"]=-1
data["nrParallel"]=-1
if self.opts.parallel:
pTimes=sol.getParallelTimes()
data["nrParallel"]=len(pTimes)
if len(pTimes)>0:
data["pFirst"]=pTimes[0]
data["pLast"]=pTimes[-1]
data["name"]=cName
data["diskusage"]=-1
if self.opts.diskusage:
data["diskusage"]=diskUsage(cName)
totalDiskusage+=data["diskusage"]
if self.opts.parallel:
for f in listdir(cName):
if re.compile("processor[0-9]+").match(f):
data["mtime"]=max(stat(path.join(cName,f))[ST_MTIME],data["mtime"])
if self.opts.state:
try:
data["nowTime"]=float(self.readState(sol,"CurrentTime"))
except ValueError:
data["nowTime"]=None
try:
data["lastOutput"]=time.mktime(time.strptime(self.readState(sol,"LastOutputSeen")))
except ValueError:
data["lastOutput"]="nix"
data["state"]=self.readState(sol,"TheState")
if self.opts.state or self.opts.estimateEndTime:
try:
data["startedAt"]=time.mktime(time.strptime(self.readState(sol,"StartedAt")))
except ValueError:
data["startedAt"]="nix"
if self.opts.startEndTime or self.opts.estimateEndTime:
try:
ctrlDict=ParsedParameterFile(sol.controlDict(),doMacroExpansion=True)
except PyFoamParserError:
# Didn't work with Macro expansion. Let's try without
try:
ctrlDict=ParsedParameterFile(sol.controlDict())
except PyFoamParserError:
ctrlDict=None
if ctrlDict:
data["startTime"]=ctrlDict["startTime"]
data["endTime"]=ctrlDict["endTime"]
else:
data["startTime"]=None
data["endTime"]=None
if self.opts.estimateEndTime:
data["endTimeEstimate"]=None
if self.readState(sol,"TheState")=="Running":
gone=time.time()-data["startedAt"]
try:
current=float(self.readState(sol,"CurrentTime"))
frac=(current-data["startTime"])/(data["endTime"]-data["startTime"])
except ValueError:
frac=0
if frac>0:
data["endTimeEstimate"]=data["startedAt"]+gone/frac
if len(customData)>0:
fn=None
pickleFile=None
if useSolverInData:
data["solver"]="none found"
# try to find the oldest pickled file
for f in ["pickledData","pickledUnfinishedData","pickledStartData"]:
dirAndTime=[]
for g in glob(path.join(cName,"*.analyzed")):
pName=path.join(g,f)
base=path.basename(g)
if base.find("PyFoamRunner.")==0:
solverName=base[len("PyFoamRunner."):-len(".analyzed")]
else:
solverName=None
if path.exists(pName):
dirAndTime.append((path.getmtime(pName),solverName,pName))
dirAndTime.sort(key=lambda x:x[0])
if len(dirAndTime)>0:
data["solver"]=dirAndTime[-1][1]
pickleFile=dirAndTime[-1][2]
break
solverName=data["solver"]
else:
solverName=self.opts.solverNameForCustom
if pickleFile:
fn=pickleFile
else:
for f in ["pickledData","pickledUnfinishedData","pickledStartData"]:
fp=path.join(cName,"PyFoamRunner."+solverName+".analyzed",f)
if path.exists(fp):
fn=fp
break
if fn:
raw=pickle.Unpickler(open(fn)).load()
for n,spec in customData:
dt=raw
for k in spec:
try:
dt=dt[k]
except KeyError:
dt="No key '"+k+"'"
break
if isinstance(dt,string_types):
break
data[n]=dt
else:
for n,spec in customData:
data[n]="no file"
cData.append(data)
except OSError:
print_(cName,"is unreadable")
if self.opts.progress:
print_("Sorting data")
cData.sort(key=lambda x:x[self.opts.sort],reverse=self.opts.reverse)
if len(cData)==0:
print_("No cases found")
return
if self.opts.dump:
print_(cData)
return
lens={}
for k in list(cData[0].keys()):
lens[k]=len(k)
for c in cData:
for k in ["mtime","lastOutput","startedAt","endTimeEstimate"]:
try:
if c[k]!=None:
if self.opts.relativeTime:
c[k]=datetime.timedelta(seconds=long(time.time()-c[k]))
else:
c[k]=time.asctime(time.localtime(c[k]))
except KeyError:
pass
except TypeError:
c[k]=None
try:
c["diskusage"]=humanReadableSize(c["diskusage"])
except KeyError:
pass
for k,v in iteritems(c):
lens[k]=max(lens[k],len(str(v)))
format=""
spec=["mtime"," | ","first"," - ","last"," (","nrSteps",") "]
if self.opts.parallel:
spec+=["| ","procs"," : ","pFirst"," - ","pLast"," (","nrParallel",") | "]
if self.opts.diskusage:
spec+=["diskusage"," | "]
if self.hasState:
spec+=["nowTime"," s ","state"," | "]
if self.opts.advancedState:
spec+=["lastOutput"," | ","startedAt"," | "]
if self.opts.estimateEndTime:
if not self.opts.advancedState:
spec+=["startedAt"," | "]
spec+=["endTimeEstimate"," | "]
if self.opts.startEndTime:
spec+=["startTime"," | ","endTime"," | "]
if useSolverInData:
spec+=["solver"," | "]
for n,s in customData:
spec+=[n," | "]
spec+=["name"]
for i,l in enumerate(spec):
if not l in list(cData[0].keys()):
format+=l
else:
if i<len(spec)-1:
format+="%%(%s)%ds" % (l,lens[l])
else:
format+="%%(%s)s" % (l)
if self.opts.progress:
print_("Printing\n\n")
header=format % dict(list(zip(list(cData[0].keys()),list(cData[0].keys()))))
print_(header)
print_("-"*len(header))
for d in cData:
for k in list(d.keys()):
d[k]=str(d[k])
print_(format % d)
if self.opts.diskusage:
print_("Total disk-usage:",humanReadableSize(totalDiskusage))
def run(self):
dirs=self.parser.getArgs()
if len(dirs)==0:
dirs=[path.curdir]
cData=[]
totalDiskusage=0
self.hasState=False
for d in dirs:
for n in listdir(d):
cName=path.join(d,n)
if path.isdir(cName):
try:
sol=SolutionDirectory(cName,archive=None,paraviewLink=False)
if sol.isValid():
if self.opts.progress:
print_("Processing",cName)
data={}
data["mtime"]=stat(cName)[ST_MTIME]
times=sol.getTimes()
try:
data["first"]=times[0]
except IndexError:
data["first"]="None"
try:
data["last"]=times[-1]
except IndexError:
data["last"]="None"
data["nrSteps"]=len(times)
data["procs"]=sol.nrProcs()
data["pFirst"]=-1
data["pLast"]=-1
data["nrParallel"]=-1
if self.opts.parallel:
pTimes=sol.getParallelTimes()
data["nrParallel"]=len(pTimes)
if len(pTimes)>0:
data["pFirst"]=pTimes[0]
data["pLast"]=pTimes[-1]
data["name"]=cName
data["diskusage"]=-1
if self.opts.diskusage:
try:
data["diskusage"]=int(
subprocess.Popen(
["du","-sb",cName],
stdout=subprocess.PIPE,
stderr=open(os.devnull,"w")
).communicate()[0].split()[0])
except IndexError:
# assume that this du does not support -b
data["diskusage"]=int(
subprocess.Popen(
["du","-sk",cName],
stdout=subprocess.PIPE
).communicate()[0].split()[0])*1024
totalDiskusage+=data["diskusage"]
if self.opts.parallel:
for f in listdir(cName):
if re.compile("processor[0-9]+").match(f):
data["mtime"]=max(stat(path.join(cName,f))[ST_MTIME],data["mtime"])
if self.opts.state:
try:
data["nowTime"]=float(self.readState(sol,"CurrentTime"))
except ValueError:
data["nowTime"]=None
try:
data["lastOutput"]=time.mktime(time.strptime(self.readState(sol,"LastOutputSeen")))
except ValueError:
data["lastOutput"]="nix"
data["state"]=self.readState(sol,"TheState")
if self.opts.state or self.opts.estimateEndTime:
try:
data["startedAt"]=time.mktime(time.strptime(self.readState(sol,"StartedAt")))
except ValueError:
data["startedAt"]="nix"
if self.opts.startEndTime or self.opts.estimateEndTime:
try:
ctrlDict=ParsedParameterFile(sol.controlDict(),doMacroExpansion=True)
except PyFoamParserError:
# Didn't work with Macro expansion. Let's try without
ctrlDict=ParsedParameterFile(sol.controlDict())
data["startTime"]=ctrlDict["startTime"]
data["endTime"]=ctrlDict["endTime"]
if self.opts.estimateEndTime:
data["endTimeEstimate"]=None
if self.readState(sol,"TheState")=="Running":
gone=time.time()-data["startedAt"]
try:
current=float(self.readState(sol,"CurrentTime"))
frac=(current-data["startTime"])/(data["endTime"]-data["startTime"])
except ValueError:
frac=0
if frac>0:
data["endTimeEstimate"]=data["startedAt"]+gone/frac
cData.append(data)
except OSError:
print_(cName,"is unreadable")
if self.opts.progress:
print_("Sorting data")
if self.opts.reverse:
cData.sort(lambda x,y:cmp(y[self.opts.sort],x[self.opts.sort]))
else:
cData.sort(lambda x,y:cmp(x[self.opts.sort],y[self.opts.sort]))
if len(cData)==0:
print_("No cases found")
return
if self.opts.dump:
print_(cData)
return
lens={}
for k in list(cData[0].keys()):
lens[k]=len(k)
for c in cData:
for k in ["mtime","lastOutput","startedAt","endTimeEstimate"]:
try:
if c[k]!=None:
if self.opts.relativeTime:
c[k]=datetime.timedelta(seconds=long(time.time()-c[k]))
else:
c[k]=time.asctime(time.localtime(c[k]))
except KeyError:
pass
except TypeError:
c[k]=None
try:
c["diskusage"]=humanReadableSize(c["diskusage"])
except KeyError:
pass
for k,v in iteritems(c):
lens[k]=max(lens[k],len(str(v)))
format=""
spec=["mtime"," | ","first"," - ","last"," (","nrSteps",") "]
if self.opts.parallel:
spec+=["| ","procs"," : ","pFirst"," - ","pLast"," (","nrParallel",") | "]
if self.opts.diskusage:
spec+=["diskusage"," | "]
if self.hasState:
spec+=["nowTime"," s ","state"," | "]
if self.opts.advancedState:
spec+=["lastOutput"," | ","startedAt"," | "]
if self.opts.estimateEndTime:
if not self.opts.advancedState:
spec+=["startedAt"," | "]
spec+=["endTimeEstimate"," | "]
if self.opts.startEndTime:
spec+=["startTime"," | ","endTime"," | "]
spec+=["name"]
for i,l in enumerate(spec):
if not l in list(cData[0].keys()):
format+=l
else:
if i<len(spec)-1:
format+="%%(%s)%ds" % (l,lens[l])
else:
format+="%%(%s)s" % (l)
if self.opts.progress:
print_("Printing\n\n")
header=format % dict(list(zip(list(cData[0].keys()),list(cData[0].keys()))))
print_(header)
print_("-"*len(header))
for d in cData:
for k in list(d.keys()):
d[k]=str(d[k])
print_(format % d)
if self.opts.diskusage:
print_("Total disk-usage:",humanReadableSize(totalDiskusage))
def doRegion(self,theRegion):
ReST=RestructuredTextHelper(defaultHeading=self.opts.headingLevel)
if self.opts.allRegions:
print_(ReST.buildHeading("Region: ",theRegion,level=self.opts.headingLevel-1))
sol=SolutionDirectory(self.parser.getArgs()[0],
archive=None,
parallel=self.opts.parallel,
paraviewLink=False,
region=theRegion)
if self.opts.all:
self.opts.caseSize=True
self.opts.shortBCreport=True
self.opts.longBCreport=True
self.opts.dimensions=True
self.opts.internal=True
self.opts.linearSolvers=True
self.opts.relaxationFactors=True
self.opts.processorMatrix=True
self.opts.decomposition=True
if self.opts.time:
try:
self.opts.time=sol.timeName(sol.timeIndex(self.opts.time,minTime=True))
except IndexError:
error("The specified time",self.opts.time,"doesn't exist in the case")
print_("Using time t="+self.opts.time+"\n")
needsPolyBoundaries=False
needsInitialTime=False
if self.opts.longBCreport:
needsPolyBoundaries=True
needsInitialTime=True
if self.opts.shortBCreport:
needsPolyBoundaries=True
needsInitialTime=True
if self.opts.dimensions:
needsInitialTime=True
if self.opts.internal:
needsInitialTime=True
if self.opts.decomposition:
needsPolyBoundaries=True
defaultProc=None
if self.opts.parallel:
defaultProc=0
if needsPolyBoundaries:
proc=None
boundary=BoundaryDict(sol.name,
region=theRegion,
time=self.opts.time,
treatBinaryAsASCII=self.opts.treatBinaryAsASCII,
processor=defaultProc)
boundMaxLen=0
boundaryNames=[]
for b in boundary:
if b.find("procBoundary")!=0:
boundaryNames.append(b)
if self.opts.patches!=None:
tmp=boundaryNames
boundaryNames=[]
for b in tmp:
for p in self.opts.patches:
if fnmatch(b,p):
boundaryNames.append(b)
break
if self.opts.expatches!=None:
tmp=boundaryNames
boundaryNames=[]
for b in tmp:
keep=True
for p in self.opts.expatches:
if fnmatch(b,p):
keep=False
break
if keep:
boundaryNames.append(b)
for b in boundaryNames:
boundMaxLen=max(boundMaxLen,len(b))
boundaryNames.sort()
if self.opts.time==None:
procTime="constant"
else:
procTime=self.opts.time
if needsInitialTime:
fields={}
if self.opts.time==None:
try:
time=sol.timeName(0)
except IndexError:
error("There is no timestep in the case")
else:
time=self.opts.time
tDir=sol[time]
nameMaxLen=0
for f in tDir:
try:
fields[f.baseName()]=f.getContent(listLengthUnparsed=self.opts.longlist,
treatBinaryAsASCII=self.opts.treatBinaryAsASCII,
doMacroExpansion=self.opts.doMacros)
nameMaxLen=max(nameMaxLen,len(f.baseName()))
except PyFoamParserError:
e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e'
warning("Couldn't parse",f.name,"because of an error:",e," -> skipping")
fieldNames=list(fields.keys())
fieldNames.sort()
if self.opts.caseSize:
print_(ReST.heading("Size of the case"))
nFaces=0
nPoints=0
nCells=0
if self.opts.parallel:
procs=list(range(sol.nrProcs()))
print_("Accumulated from",sol.nrProcs(),"processors")
else:
procs=[None]
for p in procs:
info=MeshInformation(sol.name,
processor=p,
region=theRegion,
time=self.opts.time)
nFaces+=info.nrOfFaces()
nPoints+=info.nrOfPoints()
try:
nCells+=info.nrOfCells()
except:
nCells="Not available"
tab=ReST.table()
tab[0]=("Faces",nFaces)
tab[1]=("Points",nPoints)
tab[2]=("Cells",nCells)
print_(tab)
if self.opts.decomposition:
print_(ReST.heading("Decomposition"))
if sol.nrProcs()<2:
print_("This case is not decomposed")
else:
print_("Case is decomposed for",sol.nrProcs(),"processors")
print_()
nCells=[]
nFaces=[]
nPoints=[]
for p in sol.processorDirs():
info=MeshInformation(sol.name,
processor=p,
region=theRegion,
time=self.opts.time)
nPoints.append(info.nrOfPoints())
nFaces.append(info.nrOfFaces())
nCells.append(info.nrOfCells())
digits=int(ceil(log10(max(sol.nrProcs(),
max(nCells),
max(nFaces),
max(nPoints)
))))+2
nameLen=max(len("Points"),boundMaxLen)
tab=ReST.table()
tab[0]=["CPU"]+list(range(sol.nrProcs()))
tab.addLine()
tab[1]=["Points"]+nPoints
tab[2]=["Faces"]+nFaces
tab[3]=["Cells"]+nCells
tab.addLine(head=True)
nr=3
for b in boundaryNames:
nr+=1
tab[(nr,0)]=b
for i,p in enumerate(sol.processorDirs()):
try:
nFaces= ParsedBoundaryDict(sol.boundaryDict(processor=p,
region=theRegion,
time=self.opts.time),
treatBinaryAsASCII=self.opts.treatBinaryAsASCII
)[b]["nFaces"]
except IOError:
nFaces= ParsedBoundaryDict(sol.boundaryDict(processor=p,
region=theRegion),
treatBinaryAsASCII=self.opts.treatBinaryAsASCII
)[b]["nFaces"]
except KeyError:
nFaces=0
tab[(nr,i+1)]=nFaces
print_(tab)
if self.opts.longBCreport:
print_(ReST.heading("The boundary conditions for t =",time))
for b in boundaryNames:
print_(ReST.buildHeading("Boundary: ",b,level=self.opts.headingLevel+1))
bound=boundary[b]
print_(":Type:\t",bound["type"])
if "physicalType" in bound:
print_(":Physical:\t",bound["physicalType"])
print_(":Faces:\t",bound["nFaces"])
print_()
heads=["Field","type"]
tab=ReST.table()
tab[0]=heads
tab.addLine(head=True)
for row,fName in enumerate(fieldNames):
tab[(row+1,0)]=fName
f=fields[fName]
if "boundaryField" not in f:
tab[(row+1,1)]="Not a field file"
elif b not in f["boundaryField"]:
tab[(row+1,1)]="MISSING !!!"
else:
bf=f["boundaryField"][b]
for k in bf:
try:
col=heads.index(k)
except ValueError:
col=len(heads)
tab[(0,col)]=k
heads.append(k)
cont=str(bf[k])
if type(bf[k])==Field:
if bf[k].isBinary():
cont= bf[k].binaryString()
if cont.find("\n")>=0:
tab[(row+1,col)]=cont[:cont.find("\n")]+"..."
else:
tab[(row+1,col)]=cont
print_(tab)
if self.opts.shortBCreport:
print_(ReST.heading("Table of boundary conditions for t =",time))
types={}
hasPhysical=False
for b in boundary:
if "physicalType" in boundary[b]:
hasPhysical=True
types[b]={}
for fName in fields:
f=fields[fName]
try:
if b not in f["boundaryField"]:
types[b][fName]="MISSING"
else:
types[b][fName]=f["boundaryField"][b]["type"]
except KeyError:
types[b][fName]="Not a field"
tab=ReST.table()
tab[0]=[""]+boundaryNames
tab.addLine()
tab[(1,0)]="Patch Type"
for i,b in enumerate(boundaryNames):
tab[(1,i+1)]=boundary[b]["type"]
nr=2
if hasPhysical:
tab[(nr,0)]="Physical Type"
for i,b in enumerate(boundaryNames):
if "physicalType" in boundary[b]:
tab[(nr,i+1)]=boundary[b]["physicalType"]
nr+=1
tab[(nr,0)]="Length"
for i,b in enumerate(boundaryNames):
tab[(nr,i+1)]=boundary[b]["nFaces"]
nr+=1
tab.addLine(head=True)
for fName in fieldNames:
tab[(nr,0)]=fName
for i,b in enumerate(boundaryNames):
tab[(nr,i+1)]=types[b][fName]
nr+=1
print_(tab)
if self.opts.dimensions:
print_(ReST.heading("Dimensions of fields for t =",time))
tab=ReST.table()
tab[0]=["Name"]+"[ kg m s K mol A cd ]".split()[1:-1]
tab.addLine(head=True)
for i,fName in enumerate(fieldNames):
f=fields[fName]
try:
dim=str(f["dimensions"]).split()[1:-1]
except KeyError:
dim=["-"]*7
tab[i+1]=[fName]+dim
print_(tab)
if self.opts.internal:
print_(ReST.heading("Internal value of fields for t =",time))
tab=ReST.table()
tab[0]=["Name","Value"]
tab.addLine(head=True)
for i,fName in enumerate(fieldNames):
f=fields[fName]
try:
if f["internalField"].isBinary():
val=f["internalField"].binaryString()
else:
cont=str(f["internalField"])
if cont.find("\n")>=0:
val=cont[:cont.find("\n")]+"..."
else:
val=cont
except KeyError:
val="Not a field file"
tab[i+1]=[fName,val]
print_(tab)
if self.opts.processorMatrix:
print_(ReST.heading("Processor matrix"))
if sol.nrProcs()<2:
print_("This case is not decomposed")
else:
matrix=[ [0,]*sol.nrProcs() for i in range(sol.nrProcs())]
for i,p in enumerate(sol.processorDirs()):
try:
bound=ParsedBoundaryDict(sol.boundaryDict(processor=p,
region=theRegion,
time=self.opts.time)
,treatBinaryAsASCII=self.opts.treatBinaryAsASCII)
except IOError:
bound=ParsedBoundaryDict(sol.boundaryDict(processor=p,
treatBinaryAsASCII=self.opts.treatBinaryAsASCII,
region=theRegion)
,treatBinaryAsASCII=self.opts.treatBinaryAsASCII)
for j in range(sol.nrProcs()):
name="procBoundary%dto%d" %(j,i)
name2="procBoundary%dto%d" %(i,j)
if name in bound:
matrix[i][j]=bound[name]["nFaces"]
if name2 in bound:
matrix[i][j]=bound[name2]["nFaces"]
print_("Matrix of processor interactions (faces)")
print_()
tab=ReST.table()
tab[0]=["CPU"]+list(range(sol.nrProcs()))
tab.addLine(head=True)
for i,col in enumerate(matrix):
tab[i+1]=[i]+matrix[i]
print_(tab)
if self.opts.linearSolvers:
print_(ReST.heading("Linear Solvers"))
linTable=ReST.table()
fvSol=ParsedParameterFile(path.join(sol.systemDir(),"fvSolution"),
treatBinaryAsASCII=self.opts.treatBinaryAsASCII)
allInfo={}
for sName in fvSol["solvers"]:
raw=fvSol["solvers"][sName]
info={}
if type(raw) in [dict,DictProxy]:
# fvSolution format in 1.7
info["solver"]=raw["solver"]
solverData=raw
else:
info["solver"]=raw[0]
solverData=raw[1]
if type(solverData) in [dict,DictProxy]:
try:
info["tolerance"]=solverData["tolerance"]
except KeyError:
info["tolerance"]=1.
try:
info["relTol"]=solverData["relTol"]
except KeyError:
info["relTol"]=0.
else:
# the old (pre-1.5) fvSolution-format
info["tolerance"]=solverData
info["relTol"]=raw[2]
allInfo[sName]=info
linTable[0]=["Name","Solver","Abs. Tolerance","Relative Tol."]
linTable.addLine(head=True)
nr=0
for n,i in iteritems(allInfo):
nr+=1
linTable[nr]=(n,i["solver"],i["tolerance"],i["relTol"])
print_(linTable)
if self.opts.relaxationFactors:
print_(ReST.heading("Relaxation"))
fvSol=ParsedParameterFile(path.join(sol.systemDir(),"fvSolution"),
treatBinaryAsASCII=self.opts.treatBinaryAsASCII)
if "relaxationFactors" in fvSol:
tab=ReST.table()
tab[0]=["Name","Factor"]
tab.addLine(head=True)
nr=0
for n,f in iteritems(fvSol["relaxationFactors"]):
nr+=1
tab[nr]=[n,f]
print_(tab)
else:
print_("No relaxation factors defined for this case")
def run(self):
dirs = self.parser.getArgs()
if len(dirs) == 0:
dirs = [path.curdir]
cData = []
totalDiskusage = 0
useSolverInData = False
self.hasState = False
customData = []
for i, c in enumerate(self.opts.customData):
lst = c.split("=")
if len(lst) == 2:
name, spec = lst
name += "_" # Make sure that there is no collision with standard-names
elif len(lst) == 1:
name, spec = "Custom%d" % (i + 1), c
else:
self.error("Custom specification", c,
"does not fit the pattern 'name=subs1::subs2::..'")
customData.append((name, spec.split("::")))
if len(customData) > 0 and not self.opts.solverNameForCustom:
self.warning(
"Parameter '--solver-name-for-custom-data' should be set if '--custom-data' is used"
)
useSolverInData = True
elif self.opts.hostname:
useSolverInData = True
for d in tqdm(dirs,
unit="dirs",
disable=not self.opts.progressBar or len(dirs) < 2):
if not path.isdir(d):
self.warning("There is no directory", d, "here")
continue
for n in tqdm(listdir(d),
unit="entries",
desc=path.basename(path.abspath(d)),
disable=not self.opts.progressBar):
if not self.fnmatch(n):
continue
cName = path.join(d, n)
if path.isdir(cName):
try:
sol = SolutionDirectory(cName,
archive=None,
paraviewLink=False)
if sol.isValid():
if self.opts.progress:
print_("Processing", cName)
data = {}
data["mtime"] = stat(cName)[ST_MTIME]
times = sol.getTimes()
try:
data["first"] = times[0]
except IndexError:
data["first"] = "None"
try:
data["last"] = times[-1]
except IndexError:
data["last"] = "None"
data["nrSteps"] = len(times)
data["procs"] = sol.nrProcs()
data["pFirst"] = -1
data["pLast"] = -1
data["nrParallel"] = -1
if self.opts.parallel:
pTimes = sol.getParallelTimes()
data["nrParallel"] = len(pTimes)
if len(pTimes) > 0:
data["pFirst"] = pTimes[0]
data["pLast"] = pTimes[-1]
data["name"] = cName
data["diskusage"] = -1
if self.opts.diskusage:
data["diskusage"] = diskUsage(cName)
totalDiskusage += data["diskusage"]
if self.opts.parallel:
for f in listdir(cName):
if re.compile("processor[0-9]+").match(f):
data["mtime"] = max(
stat(path.join(cName,
f))[ST_MTIME],
data["mtime"])
if self.opts.state:
try:
data["nowTime"] = float(
self.readState(sol, "CurrentTime"))
except ValueError:
data["nowTime"] = None
try:
data["lastOutput"] = time.mktime(
time.strptime(
self.readState(
sol, "LastOutputSeen")))
except ValueError:
data["lastOutput"] = "nix"
data["state"] = self.readState(sol, "TheState")
if self.opts.state or self.opts.estimateEndTime:
try:
data["startedAt"] = time.mktime(
time.strptime(
self.readState(sol, "StartedAt")))
except ValueError:
data["startedAt"] = "nix"
if self.opts.startEndTime or self.opts.estimateEndTime:
try:
ctrlDict = ParsedParameterFile(
sol.controlDict(),
doMacroExpansion=True)
except PyFoamParserError:
# Didn't work with Macro expansion. Let's try without
try:
ctrlDict = ParsedParameterFile(
sol.controlDict())
except PyFoamParserError:
ctrlDict = None
if ctrlDict:
data["startTime"] = ctrlDict["startTime"]
data["endTime"] = ctrlDict["endTime"]
else:
data["startTime"] = None
data["endTime"] = None
if self.opts.estimateEndTime:
data["endTimeEstimate"] = None
if self.readState(sol,
"TheState") == "Running":
gone = time.time() - data["startedAt"]
try:
current = float(
self.readState(sol, "CurrentTime"))
frac = (current - data["startTime"]
) / (data["endTime"] -
data["startTime"])
except ValueError:
frac = 0
if frac > 0:
data["endTimeEstimate"] = data[
"startedAt"] + gone / frac
if self.opts.hgInfo:
if path.isdir(path.join(cName, ".hg")):
from stat import ST_ATIME
prevStat = stat(cName)
try:
data["hgInfo"] = sub.Popen(
[
"hg", "id", "-R", cName, "-b",
"-n", "-i"
],
stdout=sub.PIPE).communicate(
)[0].strip()
except OSError:
data["hgInfo"] = "<hg not working>"
postStat = stat(cName)
if prevStat[ST_MTIME] != postStat[ST_MTIME]:
# hg seems to modify the modification time of the directory. So reset it
os.utime(cName, (postStat[ST_ATIME],
prevStat[ST_MTIME]))
else:
data["hgInfo"] = "<no .hg directory>"
if len(customData) > 0 or self.opts.hostname:
fn = None
pickleFile = None
if useSolverInData:
data["solver"] = "none found"
# try to find the oldest pickled file
for f in [
"pickledData",
"pickledUnfinishedData",
"pickledStartData"
]:
dirAndTime = []
for g in glob(
path.join(cName,
"*.analyzed")):
pName = path.join(g, f)
base = path.basename(g)
if base.find("PyFoamRunner.") == 0:
solverName = base[
len("PyFoamRunner."
):-len(".analyzed")]
else:
solverName = None
if path.exists(pName):
dirAndTime.append(
(path.getmtime(pName),
solverName, pName))
dirAndTime.sort(key=lambda x: x[0])
if len(dirAndTime) > 0:
data["solver"] = dirAndTime[-1][1]
pickleFile = dirAndTime[-1][2]
break
solverName = data["solver"]
else:
solverName = self.opts.solverNameForCustom
if pickleFile:
fn = pickleFile
else:
for f in [
"pickledData",
"pickledUnfinishedData",
"pickledStartData"
]:
fp = path.join(
cName, "PyFoamRunner." +
solverName + ".analyzed", f)
if path.exists(fp):
fn = fp
break
if fn:
raw = pickle.Unpickler(open(fn,
"rb")).load()
for n, spec in customData:
dt = raw
for k in spec:
try:
dt = dt[k]
except KeyError:
dt = "No key '" + k + "'"
break
if isinstance(dt, string_types):
break
data[n] = dt
if self.opts.hostname:
try:
data["hostname"] = raw[
"hostname"].split(".")[0]
except KeyError:
data["hostname"] = "<unspecified>"
else:
for n, spec in customData:
data[n] = "<no file>"
if self.opts.hostname:
data["hostname"] = "<no file>"
cData.append(data)
except OSError:
print_(cName, "is unreadable")
if self.opts.progress:
print_("Sorting data")
cData.sort(key=lambda x: x[self.opts.sort], reverse=self.opts.reverse)
if len(cData) == 0:
print_("No cases found")
return
if self.opts.dump:
print_(cData)
return
lens = {}
for k in list(cData[0].keys()):
lens[k] = len(k)
for c in cData:
for k in ["mtime", "lastOutput", "startedAt", "endTimeEstimate"]:
try:
if c[k] != None:
if self.opts.relativeTime:
c[k] = datetime.timedelta(
seconds=long(time.time() - c[k]))
else:
c[k] = time.asctime(time.localtime(c[k]))
except KeyError:
pass
except TypeError:
c[k] = None
try:
c["diskusage"] = humanReadableSize(c["diskusage"])
except KeyError:
pass
for k, v in iteritems(c):
lens[k] = max(lens[k], len(str(v)))
format = ""
spec = ["mtime", " | "]
if self.opts.hostname:
spec += ["hostname", " | "]
spec += ["first", " - ", "last", " (", "nrSteps", ") "]
if self.opts.parallel:
spec += [
"| ", "procs", " : ", "pFirst", " - ", "pLast", " (",
"nrParallel", ") | "
]
if self.opts.diskusage:
spec += ["diskusage", " | "]
if self.hasState:
spec += ["nowTime", " s ", "state", " | "]
if self.opts.advancedState:
spec += ["lastOutput", " | ", "startedAt", " | "]
if self.opts.estimateEndTime:
if not self.opts.advancedState:
spec += ["startedAt", " | "]
spec += ["endTimeEstimate", " | "]
if self.opts.startEndTime:
spec += ["startTime", " | ", "endTime", " | "]
if useSolverInData:
spec += ["solver", " | "]
for n, s in customData:
spec += [n, " | "]
if self.opts.hgInfo:
spec += ["hgInfo", " | "]
spec += ["name"]
for i, l in enumerate(spec):
if not l in list(cData[0].keys()):
format += l
else:
if i < len(spec) - 1:
format += "%%(%s)%ds" % (l, lens[l])
else:
format += "%%(%s)s" % (l)
if self.opts.progress:
print_("Printing\n\n")
header = format % dict(
list(zip(list(cData[0].keys()), list(cData[0].keys()))))
print_(header)
print_("-" * len(header))
for d in cData:
for k in list(d.keys()):
d[k] = str(d[k])
print_(format % d)
if self.opts.diskusage:
print_("Total disk-usage:", humanReadableSize(totalDiskusage))
def run(self):
dirs=self.parser.getArgs()
if len(dirs)==0:
dirs=[path.curdir]
cData=[]
totalDiskusage=0
self.hasState=False
for d in dirs:
for n in listdir(d):
cName=path.join(d,n)
if path.isdir(cName):
try:
sol=SolutionDirectory(cName,archive=None,paraviewLink=False)
if sol.isValid():
if self.opts.progress:
print "Processing",cName
data={}
data["mtime"]=stat(cName)[ST_MTIME]
times=sol.getTimes()
try:
data["first"]=times[0]
except IndexError:
data["first"]="None"
try:
data["last"]=times[-1]
except IndexError:
data["last"]="None"
data["nrSteps"]=len(times)
data["procs"]=sol.nrProcs()
data["pFirst"]=-1
data["pLast"]=-1
data["nrParallel"]=-1
if self.opts.parallel:
pTimes=sol.getParallelTimes()
data["nrParallel"]=len(pTimes)
if len(pTimes)>0:
data["pFirst"]=pTimes[0]
data["pLast"]=pTimes[-1]
data["name"]=cName
data["diskusage"]=-1
if self.opts.diskusage:
data["diskusage"]=int(subprocess.Popen(["du","-sm",cName], stdout=subprocess.PIPE).communicate()[0].split()[0])
totalDiskusage+=data["diskusage"]
if self.opts.parallel:
for f in listdir(cName):
if re.compile("processor[0-9]+").match(f):
data["mtime"]=max(stat(path.join(cName,f))[ST_MTIME],data["mtime"])
if self.opts.state:
try:
data["nowTime"]=float(self.readState(sol,"CurrentTime"))
except ValueError:
data["nowTime"]=None
try:
data["lastOutput"]=time.mktime(time.strptime(self.readState(sol,"LastOutputSeen")))
except ValueError:
data["lastOutput"]="nix"
try:
data["startedAt"]=time.mktime(time.strptime(self.readState(sol,"StartedAt")))
except ValueError:
data["startedAt"]="nix"
data["state"]=self.readState(sol,"TheState")
cData.append(data)
except OSError:
print cName,"is unreadable"
if self.opts.progress:
print "Sorting data"
if self.opts.reverse:
cData.sort(lambda x,y:cmp(y[self.opts.sort],x[self.opts.sort]))
else:
cData.sort(lambda x,y:cmp(x[self.opts.sort],y[self.opts.sort]))
if len(cData)==0:
print "No cases found"
return
if self.opts.dump:
print cData
return
lens={}
for k in cData[0].keys():
lens[k]=len(k)
for c in cData:
for k in ["mtime","lastOutput","startedAt"]:
try:
if self.opts.relativeTime:
c[k]=datetime.timedelta(seconds=long(time.time()-c[k]))
else:
c[k]=time.asctime(time.localtime(c[k]))
except KeyError:
pass
except TypeError:
c[k]=None
for k,v in c.iteritems():
lens[k]=max(lens[k],len(str(v)))
format=""
spec=["mtime"," | ","first"," - ","last"," (","nrSteps",") "]
if self.opts.parallel:
spec+=["| ","procs"," : ","pFirst"," - ","pLast"," (","nrParallel",") | "]
if self.opts.diskusage:
spec+=["diskusage"," MB "]
if self.hasState:
spec+=["nowTime"," s ","state"," | "]
if self.opts.advancedState:
spec+=["lastOutput"," | ","startedAt"," | "]
spec+=["name"]
for i,l in enumerate(spec):
if not l in cData[0].keys():
format+=l
else:
if i<len(spec)-1:
format+="%%(%s)%ds" % (l,lens[l])
else:
format+="%%(%s)s" % (l)
if self.opts.progress:
print "Printing\n\n"
header=format % dict(zip(cData[0].keys(),cData[0].keys()))
print header
print "-"*len(header)
for d in cData:
for k in d.keys():
d[k]=str(d[k])
print format % d
if self.opts.diskusage:
print "Total disk-usage:",totalDiskusage,"MB"
