def reportInstallation(self,fName):
"""Report the usages of a OpenFOAM-installation"""
print_("\nScanning",fName)
if path.islink(fName):
print_("Symlinked to",path.realpath(fName))
if not self.opts.symlinkInstallations:
print_("Skipping symlinked installation")
return 0
usages={}
self.scanDir(fName,usages)
if len(usages)>0:
nameLength=max([len(k) for k in usages.keys()])
sizeLength=max([len(str(k)) for k in usages.values()])
formatString=" %%%ds - %%%dd (%%s)" % (nameLength,sizeLength)
total=0
for k in sorted(usages.keys()):
v=usages[k]
total+=v
print_(formatString % (k,v,humanReadableSize(v)))
print_("Sum of binaries",humanReadableSize(total))
return total
else:
print_(" No binaries found")
return 0
def compressFile(self,fName):
if self.verbose>1:
print_(" Compressing",fName)
zippedName=fName+".gz"
if path.exists(zippedName):
self.warning("Zipped file",zippedName,"already existing for",fName)
return
oldSize=path.getsize(fName)
if oldSize<self.bigSize:
if self.verbose>2:
print_(" Skipping because it is too small")
self.nrSkipped+=1
return
# use gzip because that way the responsibility of removing the old file is with a 'tried and testd' program
ret=subprocess.call(["gzip",fName])
if ret!=0 or not path.exists(zippedName) or path.exists(fName):
self.warning("Problem compressing file",fName)
self.nrProblems+=1
return
newSize=path.getsize(zippedName)
if newSize>oldSize:
self.warning("Compression of",fName,"increased the filesize. Old:",
humanReadableSize(oldSize),"New:",humanReadableSize(newSize))
if self.verbose>2:
print_(" Old size:",humanReadableSize(oldSize),"New size:",humanReadableSize(newSize))
self.nrFiles+=1
self.prevSize+=oldSize
self.nowSize+=newSize
def compressFile(self, fName):
if self.verbose > 1:
print_(" Compressing", fName)
zippedName = fName + ".gz"
if path.exists(zippedName):
self.warning("Zipped file", zippedName, "already existing for",
fName)
return
oldSize = path.getsize(fName)
if oldSize < self.bigSize:
if self.verbose > 2:
print_(" Skipping because it is too small")
self.nrSkipped += 1
return
# use gzip because that way the responsibility of removing the old file is with a 'tried and testd' program
ret = subprocess.call(["gzip", fName])
if ret != 0 or not path.exists(zippedName) or path.exists(fName):
self.warning("Problem compressing file", fName)
self.nrProblems += 1
return
newSize = path.getsize(zippedName)
if newSize > oldSize:
self.warning("Compression of", fName,
"increased the filesize. Old:",
humanReadableSize(oldSize), "New:",
humanReadableSize(newSize))
if self.verbose > 2:
print_(" Old size:", humanReadableSize(oldSize), "New size:",
humanReadableSize(newSize))
self.nrFiles += 1
self.prevSize += oldSize
self.nowSize += newSize
def run(self):
dirs = self.parser.getArgs()
self.bigSize = self.opts.bigSize
self.verbose = self.opts.verbose
self.extensions = ["." + e.lower() for e in self.opts.extensions]
if self.opts.silent:
self.verbose = 0
self.opts.statistics = False
# for the statistics
self.nrDir = 0
self.nrCompressed = 0
self.nrFiles = 0
self.prevSize = 0
self.nowSize = 0
self.nrSkipped = 0
self.nrProblems = 0
try:
for d in dirs:
p = path.abspath(d)
if self.opts.recursive:
self.recursiveCompress(p)
else:
self.compressCase(p, warn=True)
except KeyboardInterrupt:
print_("Rudely interrupted by Control-C")
if self.opts.statistics:
if self.verbose > 0:
print_()
print_(self.nrDir, "case directories processed")
if self.nrFiles == 0:
print_("No files to compress found")
else:
print_(self.nrFiles, "files processed")
print_("Reduced total size from",
humanReadableSize(self.prevSize), "to",
humanReadableSize(self.nowSize), ". This is",
(100. * self.nowSize) / self.prevSize,
"percent of the original")
if self.nrSkipped > 0:
print_("Skipped", self.nrSkipped,
"files because they were smaller than", self.bigSize)
if self.nrCompressed > 0:
print_("Skipped", self.nrCompressed,
"files because they are already compressed")
if self.nrProblems > 0:
print_("Problems during the compression of", self.nrProblems,
"files")
def run(self):
dirs=self.parser.getArgs()
self.bigSize=self.opts.bigSize
self.verbose=self.opts.verbose
self.extensions=["."+e.lower() for e in self.opts.extensions]
if self.opts.silent:
self.verbose=0
self.opts.statistics=False
# for the statistics
self.nrDir=0
self.nrCompressed=0
self.nrFiles=0
self.prevSize=0
self.nowSize=0
self.nrSkipped=0
self.nrProblems=0
try:
for d in dirs:
p=path.abspath(d)
if self.opts.recursive:
self.recursiveCompress(p)
else:
self.compressCase(p,warn=True)
except KeyboardInterrupt:
print_("Rudely interrupted by Control-C")
if self.opts.statistics:
if self.verbose>0:
print_()
print_(self.nrDir,"case directories processed")
if self.nrFiles==0:
print_("No files to compress found")
else:
print_(self.nrFiles,"files processed")
print_("Reduced total size from",humanReadableSize(self.prevSize),
"to",humanReadableSize(self.nowSize),
". This is",(100.*self.nowSize)/self.prevSize,"percent of the original")
if self.nrSkipped>0:
print_("Skipped",self.nrSkipped,"files because they were smaller than",self.bigSize)
if self.nrCompressed>0:
print_("Skipped",self.nrCompressed,"files because they are already compressed")
if self.nrProblems>0:
print_("Problems during the compression of",self.nrProblems,"files")
def run(self):
if self.opts.allInstallations:
installed=FI.foamInstalledVersions()
total=0
for k in sorted(installed.keys()):
instPath=installed[k]
total+=self.reportInstallation(instPath)
print_("\nTotal disk space used by binaries:"+humanReadableSize(total))
else:
self.reportInstallation(FI.installationPath())
def compressCase(self, dirName, warn=False):
if not path.exists(dirName):
self.error("Directory", dirName, "does not exist")
s = SolutionDirectory(dirName,
archive=None,
paraviewLink=False,
parallel=True,
tolerant=True)
if not s.isValid():
if warn:
print_("Directory", dirName, "is not an OpenFOAM-case")
return
self.nrDir += 1
oldNr = self.nrFiles
oldUnc = self.prevSize
oldCon = self.nowSize
if self.verbose > 0:
print_("Processing case", dirName)
# compress meshes
for d in glob(path.join(dirName, "*", "polyMesh")) + glob(
path.join(dirName, "*", "*", "polyMesh")):
if path.isdir(d):
self.compressDirectory(d)
# compress times
for t in s:
self.compressDirectory(t.name)
# compress logfiles if requested
if self.opts.logfile:
for f in glob(path.join(dirName, "*.logfile")):
self.compressFile(path.join(dirName, f))
# processor direcories
for p in s.procDirs:
self.compressDirectory(path.join(dirName, p))
if self.nrFiles > oldNr and self.verbose > 0:
print_(
" -> ", self.nrFiles - oldNr, "files compressed.",
humanReadableSize((self.prevSize - oldUnc) -
(self.nowSize - oldCon)), "gained")
def run(self):
self.out=""
if self.opts.allInstallations:
installed=FI.foamInstalledVersions()
total=0
for k in tqdm(sorted(installed.keys()),
desc="Distro",
unit="distro"):
instPath=installed[k]
total+=self.reportInstallation(instPath)
self.output("\nTotal disk space used by binaries:"+humanReadableSize(total))
else:
try:
self.reportInstallation(FI.installationPath())
except KeyError:
self.error("No Foam-installation active. Specify one")
if len(self.out)>0:
print_("\n")
print_(self.out)
def run(self):
self.out = ""
if self.opts.allInstallations:
installed = FI.foamInstalledVersions()
total = 0
for k in tqdm(sorted(installed.keys()),
desc="Distro",
unit="distro"):
instPath = installed[k]
total += self.reportInstallation(instPath)
self.output("\nTotal disk space used by binaries:" +
humanReadableSize(total))
else:
try:
self.reportInstallation(FI.installationPath())
except KeyError:
self.error("No Foam-installation active. Specify one")
if len(self.out) > 0:
print_("\n")
print_(self.out)
def compressCase(self,dirName,warn=False):
if not path.exists(dirName):
self.error("Directory",dirName,"does not exist")
s=SolutionDirectory(dirName,
archive=None,
paraviewLink=False,
parallel=True,
tolerant=True)
if not s.isValid():
if warn:
print_("Directory",dirName,"is not an OpenFOAM-case")
return
self.nrDir+=1
oldNr=self.nrFiles
oldUnc=self.prevSize
oldCon=self.nowSize
if self.verbose>0:
print_("Processing case",dirName)
# compress meshes
for d in glob(path.join(dirName,"*","polyMesh"))+glob(path.join(dirName,"*","*","polyMesh")):
if path.isdir(d):
self.compressDirectory(d)
# compress times
for t in s:
self.compressDirectory(t.name)
# compress logfiles if requested
if self.opts.logfile:
for f in glob(path.join(dirName,"*.logfile")):
self.compressFile(path.join(dirName,f))
# processor direcories
for p in s.procDirs:
self.compressDirectory(path.join(dirName,p))
if self.nrFiles>oldNr and self.verbose>0:
print_(" -> ",self.nrFiles-oldNr,"files compressed.",
humanReadableSize((self.prevSize-oldUnc)-(self.nowSize-oldCon)),"gained")
def run(self):
dirs=self.parser.getArgs()
if len(dirs)==0:
dirs=[path.curdir]
cData=[]
totalDiskusage=0
useSolverInData=False
self.hasState=False
customData=[]
for i,c in enumerate(self.opts.customData):
lst=c.split("=")
if len(lst)==2:
name,spec=lst
name+="_" # Make sure that there is no collision with standard-names
elif len(lst)==1:
name,spec="Custom%d" % (i+1),c
else:
self.error("Custom specification",c,"does not fit the pattern 'name=subs1::subs2::..'")
customData.append((name,spec.split("::")))
if len(customData)>0 and not self.opts.solverNameForCustom:
self.warning("Parameter '--solver-name-for-custom-data' should be set if '--custom-data' is used")
useSolverInData=True
elif self.opts.hostname:
useSolverInData=True
def lookForCases(d):
for n in tqdm(listdir(d),
unit="entries",
leave=False,
desc=path.basename(path.abspath(d)),
disable=not self.opts.progressBar):
if not self.fnmatch(n):
continue
cName=path.join(d,n)
if path.isdir(cName):
try:
sol=SolutionDirectory(cName,archive=None,paraviewLink=False)
if sol.isValid():
if self.opts.progress:
print_("Processing",cName)
data={}
data["mtime"]=stat(cName)[ST_MTIME]
times=sol.getTimes()
try:
data["first"]=times[0]
except IndexError:
data["first"]="None"
try:
data["last"]=times[-1]
except IndexError:
data["last"]="None"
data["nrSteps"]=len(times)
data["procs"]=sol.nrProcs()
data["pFirst"]=-1
data["pLast"]=-1
data["nrParallel"]=-1
if self.opts.parallel:
pTimes=sol.getParallelTimes()
data["nrParallel"]=len(pTimes)
if len(pTimes)>0:
data["pFirst"]=pTimes[0]
data["pLast"]=pTimes[-1]
data["name"]=cName
data["diskusage"]=-1
if self.opts.diskusage:
data["diskusage"]=diskUsage(cName)
totalDiskusage+=data["diskusage"]
if self.opts.parallel:
for f in listdir(cName):
if re.compile("processor[0-9]+").match(f):
data["mtime"]=max(stat(path.join(cName,f))[ST_MTIME],data["mtime"])
if self.opts.state or self.opts.estimateEndTime:
try:
data["startedAt"]=time.mktime(time.strptime(self.readState(sol,"StartedAt")))
except ValueError:
data["startedAt"]="nix"
if self.opts.state:
try:
data["nowTime"]=float(self.readState(sol,"CurrentTime"))
except ValueError:
data["nowTime"]=None
try:
data["lastOutput"]=time.mktime(time.strptime(self.readState(sol,"LastOutputSeen")))
except ValueError:
data["lastOutput"]="nix"
data["state"]=self.readState(sol,"TheState")
if data["state"]=="Running":
try:
gone=time.time()-data["lastOutput"]
if gone>self.opts.deadThreshold:
data["state"]="Dead "+humanReadableDuration(gone)
except KeyError:
pass
except TypeError:
pass
if self.opts.startEndTime or self.opts.estimateEndTime:
try:
ctrlDict=ParsedParameterFile(sol.controlDict(),doMacroExpansion=True)
except PyFoamParserError:
# Didn't work with Macro expansion. Let's try without
try:
ctrlDict=ParsedParameterFile(sol.controlDict())
except PyFoamParserError:
ctrlDict=None
if ctrlDict:
data["startTime"]=ctrlDict["startTime"]
data["endTime"]=ctrlDict["endTime"]
else:
data["startTime"]=None
data["endTime"]=None
if self.opts.estimateEndTime:
data["endTimeEstimate"]=None
if self.readState(sol,"TheState")=="Running":
gone=time.time()-data["startedAt"]
try:
current=float(self.readState(sol,"CurrentTime"))
frac=(current-data["startTime"])/(data["endTime"]-data["startTime"])
except ValueError:
frac=0
if frac>0:
data["endTimeEstimate"]=data["startedAt"]+gone/frac
if self.opts.hgInfo:
if path.isdir(path.join(cName,".hg")):
from stat import ST_ATIME
prevStat=stat(cName)
try:
data["hgInfo"]=sub.Popen(["hg", "id",
"-R",cName,
"-b","-n","-i"], stdout=sub.PIPE).communicate()[0].strip()
except OSError:
data["hgInfo"]="<hg not working>"
postStat=stat(cName)
if prevStat[ST_MTIME]!=postStat[ST_MTIME]:
# hg seems to modify the modification time of the directory. So reset it
os.utime(cName,(postStat[ST_ATIME],prevStat[ST_MTIME]))
else:
data["hgInfo"]="<no .hg directory>"
if len(customData)>0 or self.opts.hostname:
fn=None
pickleFile=None
if useSolverInData:
data["solver"]="none found"
# try to find the oldest pickled file
dirAndTime=[]
for f in ["pickledData","pickledUnfinishedData","pickledStartData"]:
for g in glob(path.join(cName,"*.analyzed")):
pName=path.join(g,f)
base=path.basename(g)
if base.find("PyFoamRunner.")==0:
solverName=base[len("PyFoamRunner."):-len(".analyzed")]
else:
solverName=None
if path.exists(pName):
dirAndTime.append((path.getmtime(pName),solverName,pName))
dirAndTime.sort(key=lambda x:x[0])
if len(dirAndTime)>0:
data["solver"]=dirAndTime[-1][1]
pickleFile=dirAndTime[-1][2]
solverName=data["solver"]
else:
solverName=self.opts.solverNameForCustom
if pickleFile:
fn=pickleFile
else:
for f in ["pickledData","pickledUnfinishedData","pickledStartData"]:
fp=path.join(cName,"PyFoamRunner."+solverName+".analyzed",f)
if path.exists(fp):
fn=fp
break
pickleOK=False
if fn:
try:
raw=pickle.Unpickler(open(fn,"rb")).load()
pickleOK=True
for n,spec in customData:
dt=raw
for k in spec:
try:
dt=dt[k]
except KeyError:
dt="No key '"+k+"'"
break
if isinstance(dt,string_types):
break
data[n]=dt
if self.opts.hostname:
try:
data["hostname"]=raw["hostname"].split(".")[0]
except KeyError:
data["hostname"]="<unspecified>"
except ValueError:
pass
if not pickleOK:
for n,spec in customData:
data[n]="<no file>"
if self.opts.hostname:
data["hostname"]="<no file>"
cData.append(data)
elif self.opts.recursive:
# print("Recurse",cName)
lookForCases(cName)
except OSError:
print_(cName,"is unreadable")
for d in tqdm(dirs,
unit="dirs",
disable=not self.opts.progressBar or len(dirs)<2):
if not path.isdir(d):
self.warning("There is no directory",d,"here")
continue
lookForCases(d)
if self.opts.progress:
print_("Sorting data")
cData.sort(key=lambda x:x[self.opts.sort],reverse=self.opts.reverse)
if len(cData)==0:
print_("No cases found")
return
if self.opts.dump:
print_(cData)
return
lens={}
for k in list(cData[0].keys()):
lens[k]=len(k)
for c in cData:
for k in ["mtime","lastOutput","startedAt","endTimeEstimate"]:
try:
if c[k]!=None:
if self.opts.relativeTime:
c[k]=datetime.timedelta(seconds=long(time.time()-c[k]))
if k in ["endTimeEstimate"]:
c[k]=-c[k]
else:
c[k]=time.asctime(time.localtime(c[k]))
except KeyError:
pass
except TypeError:
c[k]=None
try:
c["diskusage"]=humanReadableSize(c["diskusage"])
except KeyError:
pass
for k,v in iteritems(c):
lens[k]=max(lens[k],len(str(v)))
format=""
spec=["mtime"," | "]
if self.opts.hostname:
spec+=["hostname"," | "]
spec+=["first"," - ","last"," (","nrSteps",") "]
if self.opts.parallel:
spec+=["| ","procs"," : ","pFirst"," - ","pLast"," (","nrParallel",") | "]
if self.opts.diskusage:
spec+=["diskusage"," | "]
if self.hasState:
spec+=["nowTime"," s ","state"," | "]
if self.opts.advancedState:
spec+=["lastOutput"," | ","startedAt"," | "]
if self.opts.estimateEndTime:
if not self.opts.advancedState:
spec+=["startedAt"," | "]
spec+=["endTimeEstimate"," | "]
if self.opts.startEndTime:
spec+=["startTime"," | ","endTime"," | "]
if useSolverInData:
spec+=["solver"," | "]
for n,s in customData:
spec+=[n," | "]
if self.opts.hgInfo:
spec+=["hgInfo"," | "]
spec+=["name"]
for i,l in enumerate(spec):
if not l in list(cData[0].keys()):
format+=l
else:
if i<len(spec)-1:
format+="%%(%s)%ds" % (l,lens[l])
else:
format+="%%(%s)s" % (l)
if self.opts.progress:
print_("Printing\n\n")
header=format % dict(list(zip(list(cData[0].keys()),list(cData[0].keys()))))
print_(header)
print_("-"*len(header))
for d in cData:
for k in list(d.keys()):
d[k]=str(d[k])
print_(format % d)
if self.opts.diskusage:
print_("Total disk-usage:",humanReadableSize(totalDiskusage))
def run(self):
dirs = self.parser.getArgs()
if len(dirs) == 0:
dirs = [path.curdir]
cData = []
totalDiskusage = 0
useSolverInData = False
self.hasState = False
customData = []
for i, c in enumerate(self.opts.customData):
lst = c.split("=")
if len(lst) == 2:
name, spec = lst
name += "_" # Make sure that there is no collision with standard-names
elif len(lst) == 1:
name, spec = "Custom%d" % (i + 1), c
else:
self.error("Custom specification", c,
"does not fit the pattern 'name=subs1::subs2::..'")
customData.append((name, spec.split("::")))
if len(customData) > 0 and not self.opts.solverNameForCustom:
self.warning(
"Parameter '--solver-name-for-custom-data' should be set if '--custom-data' is used"
)
useSolverInData = True
elif self.opts.hostname:
useSolverInData = True
def lookForCases(d):
for n in tqdm(listdir(d),
unit="entries",
leave=False,
desc=path.basename(path.abspath(d)),
disable=not self.opts.progressBar):
if not self.fnmatch(n):
continue
cName = path.join(d, n)
if path.isdir(cName):
try:
sol = SolutionDirectory(cName,
archive=None,
paraviewLink=False)
if sol.isValid():
if self.opts.progress:
print_("Processing", cName)
data = {}
data["mtime"] = stat(cName)[ST_MTIME]
times = sol.getTimes()
try:
data["first"] = times[0]
except IndexError:
data["first"] = "None"
try:
data["last"] = times[-1]
except IndexError:
data["last"] = "None"
data["nrSteps"] = len(times)
data["procs"] = sol.nrProcs()
data["pFirst"] = -1
data["pLast"] = -1
data["nrParallel"] = -1
if self.opts.parallel:
pTimes = sol.getParallelTimes()
data["nrParallel"] = len(pTimes)
if len(pTimes) > 0:
data["pFirst"] = pTimes[0]
data["pLast"] = pTimes[-1]
data["name"] = cName
data["diskusage"] = -1
if self.opts.diskusage:
data["diskusage"] = diskUsage(cName)
totalDiskusage += data["diskusage"]
if self.opts.parallel:
for f in listdir(cName):
if re.compile("processor[0-9]+").match(f):
data["mtime"] = max(
stat(path.join(cName,
f))[ST_MTIME],
data["mtime"])
if self.opts.state or self.opts.estimateEndTime:
try:
data["startedAt"] = time.mktime(
time.strptime(
self.readState(sol, "StartedAt")))
except ValueError:
data["startedAt"] = "nix"
if self.opts.state:
try:
data["nowTime"] = float(
self.readState(sol, "CurrentTime"))
except ValueError:
data["nowTime"] = None
try:
data["lastOutput"] = time.mktime(
time.strptime(
self.readState(
sol, "LastOutputSeen")))
except ValueError:
data["lastOutput"] = "nix"
data["state"] = self.readState(sol, "TheState")
if data["state"] == "Running":
try:
gone = time.time() - data["lastOutput"]
if gone > self.opts.deadThreshold:
data[
"state"] = "Dead " + humanReadableDuration(
gone)
except KeyError:
pass
except TypeError:
pass
if self.opts.startEndTime or self.opts.estimateEndTime:
try:
ctrlDict = ParsedParameterFile(
sol.controlDict(),
doMacroExpansion=True)
except PyFoamParserError:
# Didn't work with Macro expansion. Let's try without
try:
ctrlDict = ParsedParameterFile(
sol.controlDict())
except PyFoamParserError:
ctrlDict = None
if ctrlDict:
data["startTime"] = ctrlDict["startTime"]
data["endTime"] = ctrlDict["endTime"]
else:
data["startTime"] = None
data["endTime"] = None
if self.opts.estimateEndTime:
data["endTimeEstimate"] = None
if self.readState(sol,
"TheState") == "Running":
gone = time.time() - data["startedAt"]
try:
current = float(
self.readState(sol, "CurrentTime"))
frac = (current - data["startTime"]
) / (data["endTime"] -
data["startTime"])
except ValueError:
frac = 0
if frac > 0:
data["endTimeEstimate"] = data[
"startedAt"] + gone / frac
if self.opts.hgInfo:
if path.isdir(path.join(cName, ".hg")):
from stat import ST_ATIME
prevStat = stat(cName)
try:
data["hgInfo"] = sub.Popen(
[
"hg", "id", "-R", cName, "-b",
"-n", "-i"
],
stdout=sub.PIPE).communicate(
)[0].strip()
except OSError:
data["hgInfo"] = "<hg not working>"
postStat = stat(cName)
if prevStat[ST_MTIME] != postStat[ST_MTIME]:
# hg seems to modify the modification time of the directory. So reset it
os.utime(cName, (postStat[ST_ATIME],
prevStat[ST_MTIME]))
else:
data["hgInfo"] = "<no .hg directory>"
if len(customData) > 0 or self.opts.hostname:
fn = None
pickleFile = None
if useSolverInData:
data["solver"] = "none found"
# try to find the oldest pickled file
dirAndTime = []
for f in [
"pickledData",
"pickledUnfinishedData",
"pickledStartData"
]:
for g in glob(
path.join(cName,
"*.analyzed")):
pName = path.join(g, f)
base = path.basename(g)
if base.find("PyFoamRunner.") == 0:
solverName = base[
len("PyFoamRunner."
):-len(".analyzed")]
else:
solverName = None
if path.exists(pName):
dirAndTime.append(
(path.getmtime(pName),
solverName, pName))
dirAndTime.sort(key=lambda x: x[0])
if len(dirAndTime) > 0:
data["solver"] = dirAndTime[-1][1]
pickleFile = dirAndTime[-1][2]
solverName = data["solver"]
else:
solverName = self.opts.solverNameForCustom
if pickleFile:
fn = pickleFile
else:
for f in [
"pickledData",
"pickledUnfinishedData",
"pickledStartData"
]:
fp = path.join(
cName, "PyFoamRunner." +
solverName + ".analyzed", f)
if path.exists(fp):
fn = fp
break
pickleOK = False
if fn:
try:
raw = pickle.Unpickler(open(
fn, "rb")).load()
pickleOK = True
for n, spec in customData:
dt = raw
for k in spec:
try:
dt = dt[k]
except KeyError:
dt = "No key '" + k + "'"
break
if isinstance(
dt, string_types):
break
data[n] = dt
if self.opts.hostname:
try:
data["hostname"] = raw[
"hostname"].split(".")[0]
except KeyError:
data[
"hostname"] = "<unspecified>"
except ValueError:
pass
if not pickleOK:
for n, spec in customData:
data[n] = "<no file>"
if self.opts.hostname:
data["hostname"] = "<no file>"
cData.append(data)
elif self.opts.recursive:
# print("Recurse",cName)
lookForCases(cName)
except OSError:
print_(cName, "is unreadable")
for d in tqdm(dirs,
unit="dirs",
disable=not self.opts.progressBar or len(dirs) < 2):
if not path.isdir(d):
self.warning("There is no directory", d, "here")
continue
lookForCases(d)
if self.opts.progress:
print_("Sorting data")
cData.sort(key=lambda x: x[self.opts.sort], reverse=self.opts.reverse)
if len(cData) == 0:
print_("No cases found")
return
if self.opts.dump:
print_(cData)
return
lens = {}
for k in list(cData[0].keys()):
lens[k] = len(k)
for c in cData:
for k in ["mtime", "lastOutput", "startedAt", "endTimeEstimate"]:
try:
if c[k] != None:
if self.opts.relativeTime:
c[k] = datetime.timedelta(
seconds=long(time.time() - c[k]))
if k in ["endTimeEstimate"]:
c[k] = -c[k]
else:
c[k] = time.asctime(time.localtime(c[k]))
except KeyError:
pass
except TypeError:
c[k] = None
try:
c["diskusage"] = humanReadableSize(c["diskusage"])
except KeyError:
pass
for k, v in iteritems(c):
lens[k] = max(lens[k], len(str(v)))
format = ""
spec = ["mtime", " | "]
if self.opts.hostname:
spec += ["hostname", " | "]
spec += ["first", " - ", "last", " (", "nrSteps", ") "]
if self.opts.parallel:
spec += [
"| ", "procs", " : ", "pFirst", " - ", "pLast", " (",
"nrParallel", ") | "
]
if self.opts.diskusage:
spec += ["diskusage", " | "]
if self.hasState:
spec += ["nowTime", " s ", "state", " | "]
if self.opts.advancedState:
spec += ["lastOutput", " | ", "startedAt", " | "]
if self.opts.estimateEndTime:
if not self.opts.advancedState:
spec += ["startedAt", " | "]
spec += ["endTimeEstimate", " | "]
if self.opts.startEndTime:
spec += ["startTime", " | ", "endTime", " | "]
if useSolverInData:
spec += ["solver", " | "]
for n, s in customData:
spec += [n, " | "]
if self.opts.hgInfo:
spec += ["hgInfo", " | "]
spec += ["name"]
for i, l in enumerate(spec):
if not l in list(cData[0].keys()):
format += l
else:
if i < len(spec) - 1:
format += "%%(%s)%ds" % (l, lens[l])
else:
format += "%%(%s)s" % (l)
if self.opts.progress:
print_("Printing\n\n")
header = format % dict(
list(zip(list(cData[0].keys()), list(cData[0].keys()))))
print_(header)
print_("-" * len(header))
for d in cData:
for k in list(d.keys()):
d[k] = str(d[k])
print_(format % d)
if self.opts.diskusage:
print_("Total disk-usage:", humanReadableSize(totalDiskusage))
def run(self):
dirs=self.parser.getArgs()
if len(dirs)==0:
dirs=[path.curdir]
cData=[]
totalDiskusage=0
self.hasState=False
for d in dirs:
for n in listdir(d):
cName=path.join(d,n)
if path.isdir(cName):
try:
sol=SolutionDirectory(cName,archive=None,paraviewLink=False)
if sol.isValid():
if self.opts.progress:
print_("Processing",cName)
data={}
data["mtime"]=stat(cName)[ST_MTIME]
times=sol.getTimes()
try:
data["first"]=times[0]
except IndexError:
data["first"]="None"
try:
data["last"]=times[-1]
except IndexError:
data["last"]="None"
data["nrSteps"]=len(times)
data["procs"]=sol.nrProcs()
data["pFirst"]=-1
data["pLast"]=-1
data["nrParallel"]=-1
if self.opts.parallel:
pTimes=sol.getParallelTimes()
data["nrParallel"]=len(pTimes)
if len(pTimes)>0:
data["pFirst"]=pTimes[0]
data["pLast"]=pTimes[-1]
data["name"]=cName
data["diskusage"]=-1
if self.opts.diskusage:
try:
data["diskusage"]=int(
subprocess.Popen(
["du","-sb",cName],
stdout=subprocess.PIPE,
stderr=open(os.devnull,"w")
).communicate()[0].split()[0])
except IndexError:
# assume that this du does not support -b
data["diskusage"]=int(
subprocess.Popen(
["du","-sk",cName],
stdout=subprocess.PIPE
).communicate()[0].split()[0])*1024
totalDiskusage+=data["diskusage"]
if self.opts.parallel:
for f in listdir(cName):
if re.compile("processor[0-9]+").match(f):
data["mtime"]=max(stat(path.join(cName,f))[ST_MTIME],data["mtime"])
if self.opts.state:
try:
data["nowTime"]=float(self.readState(sol,"CurrentTime"))
except ValueError:
data["nowTime"]=None
try:
data["lastOutput"]=time.mktime(time.strptime(self.readState(sol,"LastOutputSeen")))
except ValueError:
data["lastOutput"]="nix"
data["state"]=self.readState(sol,"TheState")
if self.opts.state or self.opts.estimateEndTime:
try:
data["startedAt"]=time.mktime(time.strptime(self.readState(sol,"StartedAt")))
except ValueError:
data["startedAt"]="nix"
if self.opts.startEndTime or self.opts.estimateEndTime:
try:
ctrlDict=ParsedParameterFile(sol.controlDict(),doMacroExpansion=True)
except PyFoamParserError:
# Didn't work with Macro expansion. Let's try without
ctrlDict=ParsedParameterFile(sol.controlDict())
data["startTime"]=ctrlDict["startTime"]
data["endTime"]=ctrlDict["endTime"]
if self.opts.estimateEndTime:
data["endTimeEstimate"]=None
if self.readState(sol,"TheState")=="Running":
gone=time.time()-data["startedAt"]
try:
current=float(self.readState(sol,"CurrentTime"))
frac=(current-data["startTime"])/(data["endTime"]-data["startTime"])
except ValueError:
frac=0
if frac>0:
data["endTimeEstimate"]=data["startedAt"]+gone/frac
cData.append(data)
except OSError:
print_(cName,"is unreadable")
if self.opts.progress:
print_("Sorting data")
if self.opts.reverse:
cData.sort(lambda x,y:cmp(y[self.opts.sort],x[self.opts.sort]))
else:
cData.sort(lambda x,y:cmp(x[self.opts.sort],y[self.opts.sort]))
if len(cData)==0:
print_("No cases found")
return
if self.opts.dump:
print_(cData)
return
lens={}
for k in list(cData[0].keys()):
lens[k]=len(k)
for c in cData:
for k in ["mtime","lastOutput","startedAt","endTimeEstimate"]:
try:
if c[k]!=None:
if self.opts.relativeTime:
c[k]=datetime.timedelta(seconds=long(time.time()-c[k]))
else:
c[k]=time.asctime(time.localtime(c[k]))
except KeyError:
pass
except TypeError:
c[k]=None
try:
c["diskusage"]=humanReadableSize(c["diskusage"])
except KeyError:
pass
for k,v in iteritems(c):
lens[k]=max(lens[k],len(str(v)))
format=""
spec=["mtime"," | ","first"," - ","last"," (","nrSteps",") "]
if self.opts.parallel:
spec+=["| ","procs"," : ","pFirst"," - ","pLast"," (","nrParallel",") | "]
if self.opts.diskusage:
spec+=["diskusage"," | "]
if self.hasState:
spec+=["nowTime"," s ","state"," | "]
if self.opts.advancedState:
spec+=["lastOutput"," | ","startedAt"," | "]
if self.opts.estimateEndTime:
if not self.opts.advancedState:
spec+=["startedAt"," | "]
spec+=["endTimeEstimate"," | "]
if self.opts.startEndTime:
spec+=["startTime"," | ","endTime"," | "]
spec+=["name"]
for i,l in enumerate(spec):
if not l in list(cData[0].keys()):
format+=l
else:
if i<len(spec)-1:
format+="%%(%s)%ds" % (l,lens[l])
else:
format+="%%(%s)s" % (l)
if self.opts.progress:
print_("Printing\n\n")
header=format % dict(list(zip(list(cData[0].keys()),list(cData[0].keys()))))
print_(header)
print_("-"*len(header))
for d in cData:
for k in list(d.keys()):
d[k]=str(d[k])
print_(format % d)
if self.opts.diskusage:
print_("Total disk-usage:",humanReadableSize(totalDiskusage))
def run(self):
dirs = self.parser.getArgs()
if len(dirs) == 0:
dirs = [path.curdir]
cData = []
totalDiskusage = 0
useSolverInData = False
self.hasState = False
customData = []
for i, c in enumerate(self.opts.customData):
lst = c.split("=")
if len(lst) == 2:
name, spec = lst
name += "_" # Make sure that there is no collision with standard-names
elif len(lst) == 1:
name, spec = "Custom%d" % (i + 1), c
else:
self.error("Custom specification", c,
"does not fit the pattern 'name=subs1::subs2::..'")
customData.append((name, spec.split("::")))
if len(customData) > 0 and not self.opts.solverNameForCustom:
self.warning(
"Parameter '--solver-name-for-custom-data' should be set if '--custom-data' is used"
)
useSolverInData = True
for d in dirs:
for n in listdir(d):
cName = path.join(d, n)
if path.isdir(cName):
try:
sol = SolutionDirectory(cName,
archive=None,
paraviewLink=False)
if sol.isValid():
if self.opts.progress:
print_("Processing", cName)
data = {}
data["mtime"] = stat(cName)[ST_MTIME]
times = sol.getTimes()
try:
data["first"] = times[0]
except IndexError:
data["first"] = "None"
try:
data["last"] = times[-1]
except IndexError:
data["last"] = "None"
data["nrSteps"] = len(times)
data["procs"] = sol.nrProcs()
data["pFirst"] = -1
data["pLast"] = -1
data["nrParallel"] = -1
if self.opts.parallel:
pTimes = sol.getParallelTimes()
data["nrParallel"] = len(pTimes)
if len(pTimes) > 0:
data["pFirst"] = pTimes[0]
data["pLast"] = pTimes[-1]
data["name"] = cName
data["diskusage"] = -1
if self.opts.diskusage:
try:
data["diskusage"] = int(
subprocess.Popen(
["du", "-sb", cName],
stdout=subprocess.PIPE,
stderr=open(os.devnull,
"w")).communicate()
[0].split()[0])
except IndexError:
# assume that this du does not support -b
data["diskusage"] = int(
subprocess.Popen(
["du", "-sk", cName],
stdout=subprocess.PIPE).
communicate()[0].split()[0]) * 1024
totalDiskusage += data["diskusage"]
if self.opts.parallel:
for f in listdir(cName):
if re.compile("processor[0-9]+").match(f):
data["mtime"] = max(
stat(path.join(cName,
f))[ST_MTIME],
data["mtime"])
if self.opts.state:
try:
data["nowTime"] = float(
self.readState(sol, "CurrentTime"))
except ValueError:
data["nowTime"] = None
try:
data["lastOutput"] = time.mktime(
time.strptime(
self.readState(
sol, "LastOutputSeen")))
except ValueError:
data["lastOutput"] = "nix"
data["state"] = self.readState(sol, "TheState")
if self.opts.state or self.opts.estimateEndTime:
try:
data["startedAt"] = time.mktime(
time.strptime(
self.readState(sol, "StartedAt")))
except ValueError:
data["startedAt"] = "nix"
if self.opts.startEndTime or self.opts.estimateEndTime:
try:
ctrlDict = ParsedParameterFile(
sol.controlDict(),
doMacroExpansion=True)
except PyFoamParserError:
# Didn't work with Macro expansion. Let's try without
ctrlDict = ParsedParameterFile(
sol.controlDict())
data["startTime"] = ctrlDict["startTime"]
data["endTime"] = ctrlDict["endTime"]
if self.opts.estimateEndTime:
data["endTimeEstimate"] = None
if self.readState(sol,
"TheState") == "Running":
gone = time.time() - data["startedAt"]
try:
current = float(
self.readState(sol, "CurrentTime"))
frac = (current - data["startTime"]
) / (data["endTime"] -
data["startTime"])
except ValueError:
frac = 0
if frac > 0:
data["endTimeEstimate"] = data[
"startedAt"] + gone / frac
if len(customData) > 0:
fn = None
if useSolverInData:
data["solver"] = "none found"
# try to find the oldest pickled file
for f in [
"pickledData",
"pickledUnfinishedData",
"pickledStartData"
]:
dirAndTime = []
for g in glob(
path.join(cName,
"*.analyzed")):
pName = path.join(g, f)
base = path.basename(g)
if base.find("PyFoamRunner.") == 0:
solverName = base[
len("PyFoamRunner."
):-len(".analyzed")]
else:
solverName = None
if path.exists(pName):
dirAndTime.append(
(path.getmtime(pName),
solverName, pName))
dirAndTime.sort(
cmp=lambda x, y: cmp(x[0], y[0]))
if len(dirAndTime) > 0:
data["solver"] = dirAndTime[-1][1]
pickleFile = dirAndTime[-1][2]
break
solverName = data["solver"]
else:
solverName = self.opts.solverNameForCustom
pickleFile = None
if pickleFile:
fn = pickleFile
else:
for f in [
"pickledData",
"pickledUnfinishedData",
"pickledStartData"
]:
fp = path.join(
cName, "PyFoamRunner." +
solverName + ".analyzed", f)
if path.exists(fp):
fn = fp
break
if fn:
raw = pickle.Unpickler(open(fn)).load()
for n, spec in customData:
dt = raw
for k in spec:
try:
dt = dt[k]
except KeyError:
dt = "No key '" + k + "'"
break
if isinstance(dt, string_types):
break
data[n] = dt
else:
for n, spec in customData:
data[n] = "no file"
cData.append(data)
except OSError:
print_(cName, "is unreadable")
if self.opts.progress:
print_("Sorting data")
if self.opts.reverse:
cData.sort(lambda x, y: cmp(y[self.opts.sort], x[self.opts.sort]))
else:
cData.sort(lambda x, y: cmp(x[self.opts.sort], y[self.opts.sort]))
if len(cData) == 0:
print_("No cases found")
return
if self.opts.dump:
print_(cData)
return
lens = {}
for k in list(cData[0].keys()):
lens[k] = len(k)
for c in cData:
for k in ["mtime", "lastOutput", "startedAt", "endTimeEstimate"]:
try:
if c[k] != None:
if self.opts.relativeTime:
c[k] = datetime.timedelta(
seconds=long(time.time() - c[k]))
else:
c[k] = time.asctime(time.localtime(c[k]))
except KeyError:
pass
except TypeError:
c[k] = None
try:
c["diskusage"] = humanReadableSize(c["diskusage"])
except KeyError:
pass
for k, v in iteritems(c):
lens[k] = max(lens[k], len(str(v)))
format = ""
spec = ["mtime", " | ", "first", " - ", "last", " (", "nrSteps", ") "]
if self.opts.parallel:
spec += [
"| ", "procs", " : ", "pFirst", " - ", "pLast", " (",
"nrParallel", ") | "
]
if self.opts.diskusage:
spec += ["diskusage", " | "]
if self.hasState:
spec += ["nowTime", " s ", "state", " | "]
if self.opts.advancedState:
spec += ["lastOutput", " | ", "startedAt", " | "]
if self.opts.estimateEndTime:
if not self.opts.advancedState:
spec += ["startedAt", " | "]
spec += ["endTimeEstimate", " | "]
if self.opts.startEndTime:
spec += ["startTime", " | ", "endTime", " | "]
if useSolverInData:
spec += ["solver", " | "]
for n, s in customData:
spec += [n, " | "]
spec += ["name"]
for i, l in enumerate(spec):
if not l in list(cData[0].keys()):
format += l
else:
if i < len(spec) - 1:
format += "%%(%s)%ds" % (l, lens[l])
else:
format += "%%(%s)s" % (l)
if self.opts.progress:
print_("Printing\n\n")
header = format % dict(
list(zip(list(cData[0].keys()), list(cData[0].keys()))))
print_(header)
print_("-" * len(header))
for d in cData:
for k in list(d.keys()):
d[k] = str(d[k])
print_(format % d)
if self.opts.diskusage:
print_("Total disk-usage:", humanReadableSize(totalDiskusage))
def run(self):
dirs=self.parser.getArgs()
if len(dirs)==0:
dirs=[path.curdir]
cData=[]
totalDiskusage=0
useSolverInData=False
self.hasState=False
customData=[]
for i,c in enumerate(self.opts.customData):
lst=c.split("=")
if len(lst)==2:
name,spec=lst
name+="_" # Make sure that there is no collision with standard-names
elif len(lst)==1:
name,spec="Custom%d" % (i+1),c
else:
self.error("Custom specification",c,"does not fit the pattern 'name=subs1::subs2::..'")
customData.append((name,spec.split("::")))
if len(customData)>0 and not self.opts.solverNameForCustom:
self.warning("Parameter '--solver-name-for-custom-data' should be set if '--custom-data' is used")
useSolverInData=True
for d in dirs:
for n in listdir(d):
cName=path.join(d,n)
if path.isdir(cName):
try:
sol=SolutionDirectory(cName,archive=None,paraviewLink=False)
if sol.isValid():
if self.opts.progress:
print_("Processing",cName)
data={}
data["mtime"]=stat(cName)[ST_MTIME]
times=sol.getTimes()
try:
data["first"]=times[0]
except IndexError:
data["first"]="None"
try:
data["last"]=times[-1]
except IndexError:
data["last"]="None"
data["nrSteps"]=len(times)
data["procs"]=sol.nrProcs()
data["pFirst"]=-1
data["pLast"]=-1
data["nrParallel"]=-1
if self.opts.parallel:
pTimes=sol.getParallelTimes()
data["nrParallel"]=len(pTimes)
if len(pTimes)>0:
data["pFirst"]=pTimes[0]
data["pLast"]=pTimes[-1]
data["name"]=cName
data["diskusage"]=-1
if self.opts.diskusage:
try:
data["diskusage"]=int(
subprocess.Popen(
["du","-sb",cName],
stdout=subprocess.PIPE,
stderr=open(os.devnull,"w")
).communicate()[0].split()[0])
except IndexError:
# assume that this du does not support -b
data["diskusage"]=int(
subprocess.Popen(
["du","-sk",cName],
stdout=subprocess.PIPE
).communicate()[0].split()[0])*1024
totalDiskusage+=data["diskusage"]
if self.opts.parallel:
for f in listdir(cName):
if re.compile("processor[0-9]+").match(f):
data["mtime"]=max(stat(path.join(cName,f))[ST_MTIME],data["mtime"])
if self.opts.state:
try:
data["nowTime"]=float(self.readState(sol,"CurrentTime"))
except ValueError:
data["nowTime"]=None
try:
data["lastOutput"]=time.mktime(time.strptime(self.readState(sol,"LastOutputSeen")))
except ValueError:
data["lastOutput"]="nix"
data["state"]=self.readState(sol,"TheState")
if self.opts.state or self.opts.estimateEndTime:
try:
data["startedAt"]=time.mktime(time.strptime(self.readState(sol,"StartedAt")))
except ValueError:
data["startedAt"]="nix"
if self.opts.startEndTime or self.opts.estimateEndTime:
try:
ctrlDict=ParsedParameterFile(sol.controlDict(),doMacroExpansion=True)
except PyFoamParserError:
# Didn't work with Macro expansion. Let's try without
ctrlDict=ParsedParameterFile(sol.controlDict())
data["startTime"]=ctrlDict["startTime"]
data["endTime"]=ctrlDict["endTime"]
if self.opts.estimateEndTime:
data["endTimeEstimate"]=None
if self.readState(sol,"TheState")=="Running":
gone=time.time()-data["startedAt"]
try:
current=float(self.readState(sol,"CurrentTime"))
frac=(current-data["startTime"])/(data["endTime"]-data["startTime"])
except ValueError:
frac=0
if frac>0:
data["endTimeEstimate"]=data["startedAt"]+gone/frac
if len(customData)>0:
fn=None
if useSolverInData:
data["solver"]="none found"
# try to find the oldest pickled file
for f in ["pickledData","pickledUnfinishedData","pickledStartData"]:
dirAndTime=[]
for g in glob(path.join(cName,"*.analyzed")):
pName=path.join(g,f)
base=path.basename(g)
if base.find("PyFoamRunner.")==0:
solverName=base[len("PyFoamRunner."):-len(".analyzed")]
else:
solverName=None
if path.exists(pName):
dirAndTime.append((path.getmtime(pName),solverName,pName))
dirAndTime.sort(cmp=lambda x,y:cmp(x[0],y[0]))
if len(dirAndTime)>0:
data["solver"]=dirAndTime[-1][1]
pickleFile=dirAndTime[-1][2]
break
solverName=data["solver"]
else:
solverName=self.opts.solverNameForCustom
pickleFile=None
if pickleFile:
fn=pickleFile
else:
for f in ["pickledData","pickledUnfinishedData","pickledStartData"]:
fp=path.join(cName,"PyFoamRunner."+solverName+".analyzed",f)
if path.exists(fp):
fn=fp
break
if fn:
raw=pickle.Unpickler(open(fn)).load()
for n,spec in customData:
dt=raw
for k in spec:
try:
dt=dt[k]
except KeyError:
dt="No key '"+k+"'"
break
if isinstance(dt,string_types):
break
data[n]=dt
else:
for n,spec in customData:
data[n]="no file"
cData.append(data)
except OSError:
print_(cName,"is unreadable")
if self.opts.progress:
print_("Sorting data")
if self.opts.reverse:
cData.sort(lambda x,y:cmp(y[self.opts.sort],x[self.opts.sort]))
else:
cData.sort(lambda x,y:cmp(x[self.opts.sort],y[self.opts.sort]))
if len(cData)==0:
print_("No cases found")
return
if self.opts.dump:
print_(cData)
return
lens={}
for k in list(cData[0].keys()):
lens[k]=len(k)
for c in cData:
for k in ["mtime","lastOutput","startedAt","endTimeEstimate"]:
try:
if c[k]!=None:
if self.opts.relativeTime:
c[k]=datetime.timedelta(seconds=long(time.time()-c[k]))
else:
c[k]=time.asctime(time.localtime(c[k]))
except KeyError:
pass
except TypeError:
c[k]=None
try:
c["diskusage"]=humanReadableSize(c["diskusage"])
except KeyError:
pass
for k,v in iteritems(c):
lens[k]=max(lens[k],len(str(v)))
format=""
spec=["mtime"," | ","first"," - ","last"," (","nrSteps",") "]
if self.opts.parallel:
spec+=["| ","procs"," : ","pFirst"," - ","pLast"," (","nrParallel",") | "]
if self.opts.diskusage:
spec+=["diskusage"," | "]
if self.hasState:
spec+=["nowTime"," s ","state"," | "]
if self.opts.advancedState:
spec+=["lastOutput"," | ","startedAt"," | "]
if self.opts.estimateEndTime:
if not self.opts.advancedState:
spec+=["startedAt"," | "]
spec+=["endTimeEstimate"," | "]
if self.opts.startEndTime:
spec+=["startTime"," | ","endTime"," | "]
if useSolverInData:
spec+=["solver"," | "]
for n,s in customData:
spec+=[n," | "]
spec+=["name"]
for i,l in enumerate(spec):
if not l in list(cData[0].keys()):
format+=l
else:
if i<len(spec)-1:
format+="%%(%s)%ds" % (l,lens[l])
else:
format+="%%(%s)s" % (l)
if self.opts.progress:
print_("Printing\n\n")
header=format % dict(list(zip(list(cData[0].keys()),list(cData[0].keys()))))
print_(header)
print_("-"*len(header))
for d in cData:
for k in list(d.keys()):
d[k]=str(d[k])
print_(format % d)
if self.opts.diskusage:
print_("Total disk-usage:",humanReadableSize(totalDiskusage))