def lookForCases(d): for n in tqdm(listdir(d), unit="entries", leave=False, desc=path.basename(path.abspath(d)), disable=not self.opts.progressBar): if not self.fnmatch(n): continue cName=path.join(d,n) if path.isdir(cName): try: sol=SolutionDirectory(cName,archive=None,paraviewLink=False) if sol.isValid(): if self.opts.progress: print_("Processing",cName) data={} data["mtime"]=stat(cName)[ST_MTIME] times=sol.getTimes() try: data["first"]=times[0] except IndexError: data["first"]="None" try: data["last"]=times[-1] except IndexError: data["last"]="None" data["nrSteps"]=len(times) data["procs"]=sol.nrProcs() data["pFirst"]=-1 data["pLast"]=-1 data["nrParallel"]=-1 if self.opts.parallel: pTimes=sol.getParallelTimes() data["nrParallel"]=len(pTimes) if len(pTimes)>0: data["pFirst"]=pTimes[0] data["pLast"]=pTimes[-1] data["name"]=cName data["diskusage"]=-1 if self.opts.diskusage: data["diskusage"]=diskUsage(cName) totalDiskusage+=data["diskusage"] if self.opts.parallel: for f in listdir(cName): if re.compile("processor[0-9]+").match(f): data["mtime"]=max(stat(path.join(cName,f))[ST_MTIME],data["mtime"]) if self.opts.state or self.opts.estimateEndTime: try: data["startedAt"]=time.mktime(time.strptime(self.readState(sol,"StartedAt"))) except ValueError: data["startedAt"]="nix" if self.opts.state: try: data["nowTime"]=float(self.readState(sol,"CurrentTime")) except ValueError: data["nowTime"]=None try: data["lastOutput"]=time.mktime(time.strptime(self.readState(sol,"LastOutputSeen"))) except ValueError: data["lastOutput"]="nix" data["state"]=self.readState(sol,"TheState") if data["state"]=="Running": try: gone=time.time()-data["lastOutput"] if gone>self.opts.deadThreshold: data["state"]="Dead "+humanReadableDuration(gone) except KeyError: pass except TypeError: pass if self.opts.startEndTime or self.opts.estimateEndTime: try: ctrlDict=ParsedParameterFile(sol.controlDict(),doMacroExpansion=True) except PyFoamParserError: # Didn't work with Macro expansion. Let's try without try: ctrlDict=ParsedParameterFile(sol.controlDict()) except PyFoamParserError: ctrlDict=None if ctrlDict: data["startTime"]=ctrlDict["startTime"] data["endTime"]=ctrlDict["endTime"] else: data["startTime"]=None data["endTime"]=None if self.opts.estimateEndTime: data["endTimeEstimate"]=None if self.readState(sol,"TheState")=="Running": gone=time.time()-data["startedAt"] try: current=float(self.readState(sol,"CurrentTime")) frac=(current-data["startTime"])/(data["endTime"]-data["startTime"]) except ValueError: frac=0 if frac>0: data["endTimeEstimate"]=data["startedAt"]+gone/frac if self.opts.hgInfo: if path.isdir(path.join(cName,".hg")): from stat import ST_ATIME prevStat=stat(cName) try: data["hgInfo"]=sub.Popen(["hg", "id", "-R",cName, "-b","-n","-i"], stdout=sub.PIPE).communicate()[0].strip() except OSError: data["hgInfo"]="<hg not working>" postStat=stat(cName) if prevStat[ST_MTIME]!=postStat[ST_MTIME]: # hg seems to modify the modification time of the directory. So reset it os.utime(cName,(postStat[ST_ATIME],prevStat[ST_MTIME])) else: data["hgInfo"]="<no .hg directory>" if len(customData)>0 or self.opts.hostname: fn=None pickleFile=None if useSolverInData: data["solver"]="none found" # try to find the oldest pickled file dirAndTime=[] for f in ["pickledData","pickledUnfinishedData","pickledStartData"]: for g in glob(path.join(cName,"*.analyzed")): pName=path.join(g,f) base=path.basename(g) if base.find("PyFoamRunner.")==0: solverName=base[len("PyFoamRunner."):-len(".analyzed")] else: solverName=None if path.exists(pName): dirAndTime.append((path.getmtime(pName),solverName,pName)) dirAndTime.sort(key=lambda x:x[0]) if len(dirAndTime)>0: data["solver"]=dirAndTime[-1][1] pickleFile=dirAndTime[-1][2] solverName=data["solver"] else: solverName=self.opts.solverNameForCustom if pickleFile: fn=pickleFile else: for f in ["pickledData","pickledUnfinishedData","pickledStartData"]: fp=path.join(cName,"PyFoamRunner."+solverName+".analyzed",f) if path.exists(fp): fn=fp break pickleOK=False if fn: try: raw=pickle.Unpickler(open(fn,"rb")).load() pickleOK=True for n,spec in customData: dt=raw for k in spec: try: dt=dt[k] except KeyError: dt="No key '"+k+"'" break if isinstance(dt,string_types): break data[n]=dt if self.opts.hostname: try: data["hostname"]=raw["hostname"].split(".")[0] except KeyError: data["hostname"]="<unspecified>" except ValueError: pass if not pickleOK: for n,spec in customData: data[n]="<no file>" if self.opts.hostname: data["hostname"]="<no file>" cData.append(data) elif self.opts.recursive: # print("Recurse",cName) lookForCases(cName) except OSError: print_(cName,"is unreadable")
def lookForCases(d): for n in tqdm(listdir(d), unit="entries", leave=False, desc=path.basename(path.abspath(d)), disable=not self.opts.progressBar): if not self.fnmatch(n): continue cName = path.join(d, n) if path.isdir(cName): try: sol = SolutionDirectory(cName, archive=None, paraviewLink=False) if sol.isValid(): if self.opts.progress: print_("Processing", cName) data = {} data["mtime"] = stat(cName)[ST_MTIME] times = sol.getTimes() try: data["first"] = times[0] except IndexError: data["first"] = "None" try: data["last"] = times[-1] except IndexError: data["last"] = "None" data["nrSteps"] = len(times) data["procs"] = sol.nrProcs() data["pFirst"] = -1 data["pLast"] = -1 data["nrParallel"] = -1 if self.opts.parallel: pTimes = sol.getParallelTimes() data["nrParallel"] = len(pTimes) if len(pTimes) > 0: data["pFirst"] = pTimes[0] data["pLast"] = pTimes[-1] data["name"] = cName data["diskusage"] = -1 if self.opts.diskusage: data["diskusage"] = diskUsage(cName) totalDiskusage += data["diskusage"] if self.opts.parallel: for f in listdir(cName): if re.compile("processor[0-9]+").match(f): data["mtime"] = max( stat(path.join(cName, f))[ST_MTIME], data["mtime"]) if self.opts.state or self.opts.estimateEndTime: try: data["startedAt"] = time.mktime( time.strptime( self.readState(sol, "StartedAt"))) except ValueError: data["startedAt"] = "nix" if self.opts.state: try: data["nowTime"] = float( self.readState(sol, "CurrentTime")) except ValueError: data["nowTime"] = None try: data["lastOutput"] = time.mktime( time.strptime( self.readState( sol, "LastOutputSeen"))) except ValueError: data["lastOutput"] = "nix" data["state"] = self.readState(sol, "TheState") if data["state"] == "Running": try: gone = time.time() - data["lastOutput"] if gone > self.opts.deadThreshold: data[ "state"] = "Dead " + humanReadableDuration( gone) except KeyError: pass except TypeError: pass if self.opts.startEndTime or self.opts.estimateEndTime: try: ctrlDict = ParsedParameterFile( sol.controlDict(), doMacroExpansion=True) except PyFoamParserError: # Didn't work with Macro expansion. Let's try without try: ctrlDict = ParsedParameterFile( sol.controlDict()) except PyFoamParserError: ctrlDict = None if ctrlDict: data["startTime"] = ctrlDict["startTime"] data["endTime"] = ctrlDict["endTime"] else: data["startTime"] = None data["endTime"] = None if self.opts.estimateEndTime: data["endTimeEstimate"] = None if self.readState(sol, "TheState") == "Running": gone = time.time() - data["startedAt"] try: current = float( self.readState(sol, "CurrentTime")) frac = (current - data["startTime"] ) / (data["endTime"] - data["startTime"]) except ValueError: frac = 0 if frac > 0: data["endTimeEstimate"] = data[ "startedAt"] + gone / frac if self.opts.hgInfo: if path.isdir(path.join(cName, ".hg")): from stat import ST_ATIME prevStat = stat(cName) try: data["hgInfo"] = sub.Popen( [ "hg", "id", "-R", cName, "-b", "-n", "-i" ], stdout=sub.PIPE).communicate( )[0].strip() except OSError: data["hgInfo"] = "<hg not working>" postStat = stat(cName) if prevStat[ST_MTIME] != postStat[ST_MTIME]: # hg seems to modify the modification time of the directory. So reset it os.utime(cName, (postStat[ST_ATIME], prevStat[ST_MTIME])) else: data["hgInfo"] = "<no .hg directory>" if len(customData) > 0 or self.opts.hostname: fn = None pickleFile = None if useSolverInData: data["solver"] = "none found" # try to find the oldest pickled file dirAndTime = [] for f in [ "pickledData", "pickledUnfinishedData", "pickledStartData" ]: for g in glob( path.join(cName, "*.analyzed")): pName = path.join(g, f) base = path.basename(g) if base.find("PyFoamRunner.") == 0: solverName = base[ len("PyFoamRunner." ):-len(".analyzed")] else: solverName = None if path.exists(pName): dirAndTime.append( (path.getmtime(pName), solverName, pName)) dirAndTime.sort(key=lambda x: x[0]) if len(dirAndTime) > 0: data["solver"] = dirAndTime[-1][1] pickleFile = dirAndTime[-1][2] solverName = data["solver"] else: solverName = self.opts.solverNameForCustom if pickleFile: fn = pickleFile else: for f in [ "pickledData", "pickledUnfinishedData", "pickledStartData" ]: fp = path.join( cName, "PyFoamRunner." + solverName + ".analyzed", f) if path.exists(fp): fn = fp break pickleOK = False if fn: try: raw = pickle.Unpickler(open( fn, "rb")).load() pickleOK = True for n, spec in customData: dt = raw for k in spec: try: dt = dt[k] except KeyError: dt = "No key '" + k + "'" break if isinstance( dt, string_types): break data[n] = dt if self.opts.hostname: try: data["hostname"] = raw[ "hostname"].split(".")[0] except KeyError: data[ "hostname"] = "<unspecified>" except ValueError: pass if not pickleOK: for n, spec in customData: data[n] = "<no file>" if self.opts.hostname: data["hostname"] = "<no file>" cData.append(data) elif self.opts.recursive: # print("Recurse",cName) lookForCases(cName) except OSError: print_(cName, "is unreadable")
def doRegion(self, theRegion): ReST = RestructuredTextHelper(defaultHeading=self.opts.headingLevel) if self.opts.allRegions: print_( ReST.buildHeading("Region: ", theRegion, level=self.opts.headingLevel - 1)) sol = SolutionDirectory(self.parser.getArgs()[0], archive=None, parallel=self.opts.parallel, paraviewLink=False, region=theRegion) if self.opts.all: self.opts.caseSize = True self.opts.shortBCreport = True self.opts.longBCreport = True self.opts.dimensions = True self.opts.internal = True self.opts.linearSolvers = True self.opts.relaxationFactors = True self.opts.processorMatrix = True self.opts.decomposition = True if self.opts.time: try: self.opts.time = sol.timeName( sol.timeIndex(self.opts.time, minTime=True)) except IndexError: error("The specified time", self.opts.time, "doesn't exist in the case") print_("Using time t=" + self.opts.time + "\n") needsPolyBoundaries = False needsInitialTime = False if self.opts.longBCreport: needsPolyBoundaries = True needsInitialTime = True if self.opts.shortBCreport: needsPolyBoundaries = True needsInitialTime = True if self.opts.dimensions: needsInitialTime = True if self.opts.internal: needsInitialTime = True if self.opts.decomposition: needsPolyBoundaries = True defaultProc = None if self.opts.parallel: defaultProc = 0 if needsPolyBoundaries: proc = None boundary = BoundaryDict( sol.name, region=theRegion, time=self.opts.time, treatBinaryAsASCII=self.opts.boundaryTreatBinaryAsASCII, processor=defaultProc) boundMaxLen = 0 boundaryNames = [] for b in boundary: if b.find("procBoundary") != 0: boundaryNames.append(b) if self.opts.patches != None: tmp = boundaryNames boundaryNames = [] for b in tmp: for p in self.opts.patches: if fnmatch(b, p): boundaryNames.append(b) break if self.opts.expatches != None: tmp = boundaryNames boundaryNames = [] for b in tmp: keep = True for p in self.opts.expatches: if fnmatch(b, p): keep = False break if keep: boundaryNames.append(b) for b in boundaryNames: boundMaxLen = max(boundMaxLen, len(b)) boundaryNames.sort() if self.opts.time == None: procTime = "constant" else: procTime = self.opts.time if needsInitialTime: fields = {} if self.opts.time == None: try: time = sol.timeName(0) except IndexError: error("There is no timestep in the case") else: time = self.opts.time tDir = sol[time] nameMaxLen = 0 for f in tDir: try: fields[f.baseName()] = f.getContent( listLengthUnparsed=self.opts.longlist, treatBinaryAsASCII=self.opts.treatBinaryAsASCII, doMacroExpansion=self.opts.doMacros) nameMaxLen = max(nameMaxLen, len(f.baseName())) except PyFoamParserError: e = sys.exc_info()[ 1] # Needed because python 2.5 does not support 'as e' warning("Couldn't parse", f.name, "because of an error:", e, " -> skipping") fieldNames = list(fields.keys()) fieldNames.sort() if self.opts.caseSize: print_(ReST.heading("Size of the case")) nFaces = 0 nPoints = 0 nCells = 0 if self.opts.parallel: procs = list(range(sol.nrProcs())) print_("Accumulated from", sol.nrProcs(), "processors") else: procs = [None] for p in procs: info = MeshInformation(sol.name, processor=p, region=theRegion, time=self.opts.time) nFaces += info.nrOfFaces() nPoints += info.nrOfPoints() try: nCells += info.nrOfCells() except: nCells = "Not available" tab = ReST.table() tab[0] = ("Faces", nFaces) tab[1] = ("Points", nPoints) tab[2] = ("Cells", nCells) print_(tab) if self.opts.decomposition: print_(ReST.heading("Decomposition")) if sol.nrProcs() < 2: print_("This case is not decomposed") else: print_("Case is decomposed for", sol.nrProcs(), "processors") print_() nCells = [] nFaces = [] nPoints = [] for p in sol.processorDirs(): info = MeshInformation(sol.name, processor=p, region=theRegion, time=self.opts.time) nPoints.append(info.nrOfPoints()) nFaces.append(info.nrOfFaces()) nCells.append(info.nrOfCells()) digits = int( ceil( log10( max(sol.nrProcs(), max(nCells), max(nFaces), max(nPoints))))) + 2 nameLen = max(len("Points"), boundMaxLen) tab = ReST.table() tab[0] = ["CPU"] + list(range(sol.nrProcs())) tab.addLine() tab[1] = ["Points"] + nPoints tab[2] = ["Faces"] + nFaces tab[3] = ["Cells"] + nCells tab.addLine(head=True) nr = 3 for b in boundaryNames: nr += 1 tab[(nr, 0)] = b for i, p in enumerate(sol.processorDirs()): try: nFaces = ParsedBoundaryDict( sol.boundaryDict(processor=p, region=theRegion, time=self.opts.time), treatBinaryAsASCII=self.opts. boundaryTreatBinaryAsASCII)[b]["nFaces"] except IOError: nFaces = ParsedBoundaryDict( sol.boundaryDict(processor=p, region=theRegion), treatBinaryAsASCII=self.opts. boundaryTreatBinaryAsASCII)[b]["nFaces"] except KeyError: nFaces = 0 tab[(nr, i + 1)] = nFaces print_(tab) if self.opts.longBCreport: print_(ReST.heading("The boundary conditions for t =", time)) for b in boundaryNames: print_( ReST.buildHeading("Boundary: ", b, level=self.opts.headingLevel + 1)) bound = boundary[b] print_(":Type:\t", bound["type"]) if "physicalType" in bound: print_(":Physical:\t", bound["physicalType"]) print_(":Faces:\t", bound["nFaces"]) print_() heads = ["Field", "type"] tab = ReST.table() tab[0] = heads tab.addLine(head=True) for row, fName in enumerate(fieldNames): tab[(row + 1, 0)] = fName f = fields[fName] if "boundaryField" not in f: tab[(row + 1, 1)] = "Not a field file" elif b not in f["boundaryField"]: tab[(row + 1, 1)] = "MISSING !!!" else: bf = f["boundaryField"][b] for k in bf: try: col = heads.index(k) except ValueError: col = len(heads) tab[(0, col)] = k heads.append(k) cont = str(bf[k]) if type(bf[k]) == Field: if bf[k].isBinary(): cont = bf[k].binaryString() if cont.find("\n") >= 0: tab[(row + 1, col)] = cont[:cont.find("\n")] + "..." else: tab[(row + 1, col)] = cont print_(tab) if self.opts.shortBCreport: print_(ReST.heading("Table of boundary conditions for t =", time)) types = {} hasPhysical = False for b in boundary: if "physicalType" in boundary[b]: hasPhysical = True types[b] = {} for fName in fields: f = fields[fName] try: if b not in f["boundaryField"]: types[b][fName] = "MISSING" else: types[b][fName] = f["boundaryField"][b]["type"] except KeyError: types[b][fName] = "Not a field" tab = ReST.table() tab[0] = [""] + boundaryNames tab.addLine() tab[(1, 0)] = "Patch Type" for i, b in enumerate(boundaryNames): tab[(1, i + 1)] = boundary[b]["type"] nr = 2 if hasPhysical: tab[(nr, 0)] = "Physical Type" for i, b in enumerate(boundaryNames): if "physicalType" in boundary[b]: tab[(nr, i + 1)] = boundary[b]["physicalType"] nr += 1 tab[(nr, 0)] = "Length" for i, b in enumerate(boundaryNames): tab[(nr, i + 1)] = boundary[b]["nFaces"] nr += 1 tab.addLine(head=True) for fName in fieldNames: tab[(nr, 0)] = fName for i, b in enumerate(boundaryNames): tab[(nr, i + 1)] = types[b][fName] nr += 1 print_(tab) if self.opts.dimensions: print_(ReST.heading("Dimensions of fields for t =", time)) tab = ReST.table() tab[0] = ["Name"] + "[ kg m s K mol A cd ]".split()[1:-1] tab.addLine(head=True) for i, fName in enumerate(fieldNames): f = fields[fName] try: dim = str(f["dimensions"]).split()[1:-1] except KeyError: dim = ["-"] * 7 tab[i + 1] = [fName] + dim print_(tab) if self.opts.internal: print_(ReST.heading("Internal value of fields for t =", time)) tab = ReST.table() tab[0] = ["Name", "Value"] tab.addLine(head=True) for i, fName in enumerate(fieldNames): f = fields[fName] try: if f["internalField"].isBinary(): val = f["internalField"].binaryString() else: cont = str(f["internalField"]) if cont.find("\n") >= 0: val = cont[:cont.find("\n")] + "..." else: val = cont except KeyError: val = "Not a field file" tab[i + 1] = [fName, val] print_(tab) if self.opts.processorMatrix: print_(ReST.heading("Processor matrix")) if sol.nrProcs() < 2: print_("This case is not decomposed") else: matrix = [[ 0, ] * sol.nrProcs() for i in range(sol.nrProcs())] for i, p in enumerate(sol.processorDirs()): try: bound = ParsedBoundaryDict( sol.boundaryDict(processor=p, region=theRegion, time=self.opts.time), treatBinaryAsASCII=self.opts. boundaryTreatBinaryAsASCII) except IOError: bound = ParsedBoundaryDict( sol.boundaryDict(processor=p, treatBinaryAsASCII=self.opts. boundaryTreatBinaryAsASCII, region=theRegion), treatBinaryAsASCII=self.opts. boundaryTreatBinaryAsASCII) for j in range(sol.nrProcs()): name = "procBoundary%dto%d" % (j, i) name2 = "procBoundary%dto%d" % (i, j) if name in bound: matrix[i][j] = bound[name]["nFaces"] if name2 in bound: matrix[i][j] = bound[name2]["nFaces"] print_("Matrix of processor interactions (faces)") print_() tab = ReST.table() tab[0] = ["CPU"] + list(range(sol.nrProcs())) tab.addLine(head=True) for i, col in enumerate(matrix): tab[i + 1] = [i] + matrix[i] print_(tab) if self.opts.linearSolvers: print_(ReST.heading("Linear Solvers")) linTable = ReST.table() fvSol = ParsedParameterFile( path.join(sol.systemDir(), "fvSolution"), treatBinaryAsASCII=self.opts.treatBinaryAsASCII) allInfo = {} for sName in fvSol["solvers"]: raw = fvSol["solvers"][sName] info = {} if type(raw) in [dict, DictProxy]: # fvSolution format in 1.7 try: info["solver"] = raw["solver"] except KeyError: info["solver"] = "<none>" solverData = raw else: info["solver"] = raw[0] solverData = raw[1] if type(solverData) in [dict, DictProxy]: try: info["tolerance"] = solverData["tolerance"] except KeyError: info["tolerance"] = 1. try: info["relTol"] = solverData["relTol"] except KeyError: info["relTol"] = 0. else: # the old (pre-1.5) fvSolution-format info["tolerance"] = solverData info["relTol"] = raw[2] allInfo[sName] = info linTable[0] = ["Name", "Solver", "Abs. Tolerance", "Relative Tol."] linTable.addLine(head=True) nr = 0 for n, i in iteritems(allInfo): nr += 1 linTable[nr] = (n, i["solver"], i["tolerance"], i["relTol"]) print_(linTable) if self.opts.relaxationFactors: print_(ReST.heading("Relaxation")) fvSol = ParsedParameterFile( path.join(sol.systemDir(), "fvSolution"), treatBinaryAsASCII=self.opts.treatBinaryAsASCII) if "relaxationFactors" in fvSol: relax = fvSol["relaxationFactors"] tab = ReST.table() tab[0] = ["Name", "Factor"] tab.addLine(head=True) nr = 0 if "fields" in relax or "equations" in relax: # New syntax for k in ["fields", "equations"]: if k in relax: for n, f in iteritems(relax[k]): nr += 1 tab[nr] = [k + ": " + n, f] else: for n, f in iteritems(relax): nr += 1 tab[nr] = [n, f] print_(tab) else: print_("No relaxation factors defined for this case")
def run(self): dirs=self.parser.getArgs() if len(dirs)==0: dirs=[path.curdir] cData=[] totalDiskusage=0 useSolverInData=False self.hasState=False customData=[] for i,c in enumerate(self.opts.customData): lst=c.split("=") if len(lst)==2: name,spec=lst name+="_" # Make sure that there is no collision with standard-names elif len(lst)==1: name,spec="Custom%d" % (i+1),c else: self.error("Custom specification",c,"does not fit the pattern 'name=subs1::subs2::..'") customData.append((name,spec.split("::"))) if len(customData)>0 and not self.opts.solverNameForCustom: self.warning("Parameter '--solver-name-for-custom-data' should be set if '--custom-data' is used") useSolverInData=True for d in dirs: for n in listdir(d): cName=path.join(d,n) if path.isdir(cName): try: sol=SolutionDirectory(cName,archive=None,paraviewLink=False) if sol.isValid(): if self.opts.progress: print_("Processing",cName) data={} data["mtime"]=stat(cName)[ST_MTIME] times=sol.getTimes() try: data["first"]=times[0] except IndexError: data["first"]="None" try: data["last"]=times[-1] except IndexError: data["last"]="None" data["nrSteps"]=len(times) data["procs"]=sol.nrProcs() data["pFirst"]=-1 data["pLast"]=-1 data["nrParallel"]=-1 if self.opts.parallel: pTimes=sol.getParallelTimes() data["nrParallel"]=len(pTimes) if len(pTimes)>0: data["pFirst"]=pTimes[0] data["pLast"]=pTimes[-1] data["name"]=cName data["diskusage"]=-1 if self.opts.diskusage: data["diskusage"]=diskUsage(cName) totalDiskusage+=data["diskusage"] if self.opts.parallel: for f in listdir(cName): if re.compile("processor[0-9]+").match(f): data["mtime"]=max(stat(path.join(cName,f))[ST_MTIME],data["mtime"]) if self.opts.state: try: data["nowTime"]=float(self.readState(sol,"CurrentTime")) except ValueError: data["nowTime"]=None try: data["lastOutput"]=time.mktime(time.strptime(self.readState(sol,"LastOutputSeen"))) except ValueError: data["lastOutput"]="nix" data["state"]=self.readState(sol,"TheState") if self.opts.state or self.opts.estimateEndTime: try: data["startedAt"]=time.mktime(time.strptime(self.readState(sol,"StartedAt"))) except ValueError: data["startedAt"]="nix" if self.opts.startEndTime or self.opts.estimateEndTime: try: ctrlDict=ParsedParameterFile(sol.controlDict(),doMacroExpansion=True) except PyFoamParserError: # Didn't work with Macro expansion. Let's try without try: ctrlDict=ParsedParameterFile(sol.controlDict()) except PyFoamParserError: ctrlDict=None if ctrlDict: data["startTime"]=ctrlDict["startTime"] data["endTime"]=ctrlDict["endTime"] else: data["startTime"]=None data["endTime"]=None if self.opts.estimateEndTime: data["endTimeEstimate"]=None if self.readState(sol,"TheState")=="Running": gone=time.time()-data["startedAt"] try: current=float(self.readState(sol,"CurrentTime")) frac=(current-data["startTime"])/(data["endTime"]-data["startTime"]) except ValueError: frac=0 if frac>0: data["endTimeEstimate"]=data["startedAt"]+gone/frac if len(customData)>0: fn=None pickleFile=None if useSolverInData: data["solver"]="none found" # try to find the oldest pickled file for f in ["pickledData","pickledUnfinishedData","pickledStartData"]: dirAndTime=[] for g in glob(path.join(cName,"*.analyzed")): pName=path.join(g,f) base=path.basename(g) if base.find("PyFoamRunner.")==0: solverName=base[len("PyFoamRunner."):-len(".analyzed")] else: solverName=None if path.exists(pName): dirAndTime.append((path.getmtime(pName),solverName,pName)) dirAndTime.sort(key=lambda x:x[0]) if len(dirAndTime)>0: data["solver"]=dirAndTime[-1][1] pickleFile=dirAndTime[-1][2] break solverName=data["solver"] else: solverName=self.opts.solverNameForCustom if pickleFile: fn=pickleFile else: for f in ["pickledData","pickledUnfinishedData","pickledStartData"]: fp=path.join(cName,"PyFoamRunner."+solverName+".analyzed",f) if path.exists(fp): fn=fp break if fn: raw=pickle.Unpickler(open(fn)).load() for n,spec in customData: dt=raw for k in spec: try: dt=dt[k] except KeyError: dt="No key '"+k+"'" break if isinstance(dt,string_types): break data[n]=dt else: for n,spec in customData: data[n]="no file" cData.append(data) except OSError: print_(cName,"is unreadable") if self.opts.progress: print_("Sorting data") cData.sort(key=lambda x:x[self.opts.sort],reverse=self.opts.reverse) if len(cData)==0: print_("No cases found") return if self.opts.dump: print_(cData) return lens={} for k in list(cData[0].keys()): lens[k]=len(k) for c in cData: for k in ["mtime","lastOutput","startedAt","endTimeEstimate"]: try: if c[k]!=None: if self.opts.relativeTime: c[k]=datetime.timedelta(seconds=long(time.time()-c[k])) else: c[k]=time.asctime(time.localtime(c[k])) except KeyError: pass except TypeError: c[k]=None try: c["diskusage"]=humanReadableSize(c["diskusage"]) except KeyError: pass for k,v in iteritems(c): lens[k]=max(lens[k],len(str(v))) format="" spec=["mtime"," | ","first"," - ","last"," (","nrSteps",") "] if self.opts.parallel: spec+=["| ","procs"," : ","pFirst"," - ","pLast"," (","nrParallel",") | "] if self.opts.diskusage: spec+=["diskusage"," | "] if self.hasState: spec+=["nowTime"," s ","state"," | "] if self.opts.advancedState: spec+=["lastOutput"," | ","startedAt"," | "] if self.opts.estimateEndTime: if not self.opts.advancedState: spec+=["startedAt"," | "] spec+=["endTimeEstimate"," | "] if self.opts.startEndTime: spec+=["startTime"," | ","endTime"," | "] if useSolverInData: spec+=["solver"," | "] for n,s in customData: spec+=[n," | "] spec+=["name"] for i,l in enumerate(spec): if not l in list(cData[0].keys()): format+=l else: if i<len(spec)-1: format+="%%(%s)%ds" % (l,lens[l]) else: format+="%%(%s)s" % (l) if self.opts.progress: print_("Printing\n\n") header=format % dict(list(zip(list(cData[0].keys()),list(cData[0].keys())))) print_(header) print_("-"*len(header)) for d in cData: for k in list(d.keys()): d[k]=str(d[k]) print_(format % d) if self.opts.diskusage: print_("Total disk-usage:",humanReadableSize(totalDiskusage))
def run(self): dirs=self.parser.getArgs() if len(dirs)==0: dirs=[path.curdir] cData=[] totalDiskusage=0 self.hasState=False for d in dirs: for n in listdir(d): cName=path.join(d,n) if path.isdir(cName): try: sol=SolutionDirectory(cName,archive=None,paraviewLink=False) if sol.isValid(): if self.opts.progress: print_("Processing",cName) data={} data["mtime"]=stat(cName)[ST_MTIME] times=sol.getTimes() try: data["first"]=times[0] except IndexError: data["first"]="None" try: data["last"]=times[-1] except IndexError: data["last"]="None" data["nrSteps"]=len(times) data["procs"]=sol.nrProcs() data["pFirst"]=-1 data["pLast"]=-1 data["nrParallel"]=-1 if self.opts.parallel: pTimes=sol.getParallelTimes() data["nrParallel"]=len(pTimes) if len(pTimes)>0: data["pFirst"]=pTimes[0] data["pLast"]=pTimes[-1] data["name"]=cName data["diskusage"]=-1 if self.opts.diskusage: try: data["diskusage"]=int( subprocess.Popen( ["du","-sb",cName], stdout=subprocess.PIPE, stderr=open(os.devnull,"w") ).communicate()[0].split()[0]) except IndexError: # assume that this du does not support -b data["diskusage"]=int( subprocess.Popen( ["du","-sk",cName], stdout=subprocess.PIPE ).communicate()[0].split()[0])*1024 totalDiskusage+=data["diskusage"] if self.opts.parallel: for f in listdir(cName): if re.compile("processor[0-9]+").match(f): data["mtime"]=max(stat(path.join(cName,f))[ST_MTIME],data["mtime"]) if self.opts.state: try: data["nowTime"]=float(self.readState(sol,"CurrentTime")) except ValueError: data["nowTime"]=None try: data["lastOutput"]=time.mktime(time.strptime(self.readState(sol,"LastOutputSeen"))) except ValueError: data["lastOutput"]="nix" data["state"]=self.readState(sol,"TheState") if self.opts.state or self.opts.estimateEndTime: try: data["startedAt"]=time.mktime(time.strptime(self.readState(sol,"StartedAt"))) except ValueError: data["startedAt"]="nix" if self.opts.startEndTime or self.opts.estimateEndTime: try: ctrlDict=ParsedParameterFile(sol.controlDict(),doMacroExpansion=True) except PyFoamParserError: # Didn't work with Macro expansion. Let's try without ctrlDict=ParsedParameterFile(sol.controlDict()) data["startTime"]=ctrlDict["startTime"] data["endTime"]=ctrlDict["endTime"] if self.opts.estimateEndTime: data["endTimeEstimate"]=None if self.readState(sol,"TheState")=="Running": gone=time.time()-data["startedAt"] try: current=float(self.readState(sol,"CurrentTime")) frac=(current-data["startTime"])/(data["endTime"]-data["startTime"]) except ValueError: frac=0 if frac>0: data["endTimeEstimate"]=data["startedAt"]+gone/frac cData.append(data) except OSError: print_(cName,"is unreadable") if self.opts.progress: print_("Sorting data") if self.opts.reverse: cData.sort(lambda x,y:cmp(y[self.opts.sort],x[self.opts.sort])) else: cData.sort(lambda x,y:cmp(x[self.opts.sort],y[self.opts.sort])) if len(cData)==0: print_("No cases found") return if self.opts.dump: print_(cData) return lens={} for k in list(cData[0].keys()): lens[k]=len(k) for c in cData: for k in ["mtime","lastOutput","startedAt","endTimeEstimate"]: try: if c[k]!=None: if self.opts.relativeTime: c[k]=datetime.timedelta(seconds=long(time.time()-c[k])) else: c[k]=time.asctime(time.localtime(c[k])) except KeyError: pass except TypeError: c[k]=None try: c["diskusage"]=humanReadableSize(c["diskusage"]) except KeyError: pass for k,v in iteritems(c): lens[k]=max(lens[k],len(str(v))) format="" spec=["mtime"," | ","first"," - ","last"," (","nrSteps",") "] if self.opts.parallel: spec+=["| ","procs"," : ","pFirst"," - ","pLast"," (","nrParallel",") | "] if self.opts.diskusage: spec+=["diskusage"," | "] if self.hasState: spec+=["nowTime"," s ","state"," | "] if self.opts.advancedState: spec+=["lastOutput"," | ","startedAt"," | "] if self.opts.estimateEndTime: if not self.opts.advancedState: spec+=["startedAt"," | "] spec+=["endTimeEstimate"," | "] if self.opts.startEndTime: spec+=["startTime"," | ","endTime"," | "] spec+=["name"] for i,l in enumerate(spec): if not l in list(cData[0].keys()): format+=l else: if i<len(spec)-1: format+="%%(%s)%ds" % (l,lens[l]) else: format+="%%(%s)s" % (l) if self.opts.progress: print_("Printing\n\n") header=format % dict(list(zip(list(cData[0].keys()),list(cData[0].keys())))) print_(header) print_("-"*len(header)) for d in cData: for k in list(d.keys()): d[k]=str(d[k]) print_(format % d) if self.opts.diskusage: print_("Total disk-usage:",humanReadableSize(totalDiskusage))
def doRegion(self,theRegion): ReST=RestructuredTextHelper(defaultHeading=self.opts.headingLevel) if self.opts.allRegions: print_(ReST.buildHeading("Region: ",theRegion,level=self.opts.headingLevel-1)) sol=SolutionDirectory(self.parser.getArgs()[0], archive=None, parallel=self.opts.parallel, paraviewLink=False, region=theRegion) if self.opts.all: self.opts.caseSize=True self.opts.shortBCreport=True self.opts.longBCreport=True self.opts.dimensions=True self.opts.internal=True self.opts.linearSolvers=True self.opts.relaxationFactors=True self.opts.processorMatrix=True self.opts.decomposition=True if self.opts.time: try: self.opts.time=sol.timeName(sol.timeIndex(self.opts.time,minTime=True)) except IndexError: error("The specified time",self.opts.time,"doesn't exist in the case") print_("Using time t="+self.opts.time+"\n") needsPolyBoundaries=False needsInitialTime=False if self.opts.longBCreport: needsPolyBoundaries=True needsInitialTime=True if self.opts.shortBCreport: needsPolyBoundaries=True needsInitialTime=True if self.opts.dimensions: needsInitialTime=True if self.opts.internal: needsInitialTime=True if self.opts.decomposition: needsPolyBoundaries=True defaultProc=None if self.opts.parallel: defaultProc=0 if needsPolyBoundaries: proc=None boundary=BoundaryDict(sol.name, region=theRegion, time=self.opts.time, treatBinaryAsASCII=self.opts.treatBinaryAsASCII, processor=defaultProc) boundMaxLen=0 boundaryNames=[] for b in boundary: if b.find("procBoundary")!=0: boundaryNames.append(b) if self.opts.patches!=None: tmp=boundaryNames boundaryNames=[] for b in tmp: for p in self.opts.patches: if fnmatch(b,p): boundaryNames.append(b) break if self.opts.expatches!=None: tmp=boundaryNames boundaryNames=[] for b in tmp: keep=True for p in self.opts.expatches: if fnmatch(b,p): keep=False break if keep: boundaryNames.append(b) for b in boundaryNames: boundMaxLen=max(boundMaxLen,len(b)) boundaryNames.sort() if self.opts.time==None: procTime="constant" else: procTime=self.opts.time if needsInitialTime: fields={} if self.opts.time==None: try: time=sol.timeName(0) except IndexError: error("There is no timestep in the case") else: time=self.opts.time tDir=sol[time] nameMaxLen=0 for f in tDir: try: fields[f.baseName()]=f.getContent(listLengthUnparsed=self.opts.longlist, treatBinaryAsASCII=self.opts.treatBinaryAsASCII, doMacroExpansion=self.opts.doMacros) nameMaxLen=max(nameMaxLen,len(f.baseName())) except PyFoamParserError: e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e' warning("Couldn't parse",f.name,"because of an error:",e," -> skipping") fieldNames=list(fields.keys()) fieldNames.sort() if self.opts.caseSize: print_(ReST.heading("Size of the case")) nFaces=0 nPoints=0 nCells=0 if self.opts.parallel: procs=list(range(sol.nrProcs())) print_("Accumulated from",sol.nrProcs(),"processors") else: procs=[None] for p in procs: info=MeshInformation(sol.name, processor=p, region=theRegion, time=self.opts.time) nFaces+=info.nrOfFaces() nPoints+=info.nrOfPoints() try: nCells+=info.nrOfCells() except: nCells="Not available" tab=ReST.table() tab[0]=("Faces",nFaces) tab[1]=("Points",nPoints) tab[2]=("Cells",nCells) print_(tab) if self.opts.decomposition: print_(ReST.heading("Decomposition")) if sol.nrProcs()<2: print_("This case is not decomposed") else: print_("Case is decomposed for",sol.nrProcs(),"processors") print_() nCells=[] nFaces=[] nPoints=[] for p in sol.processorDirs(): info=MeshInformation(sol.name, processor=p, region=theRegion, time=self.opts.time) nPoints.append(info.nrOfPoints()) nFaces.append(info.nrOfFaces()) nCells.append(info.nrOfCells()) digits=int(ceil(log10(max(sol.nrProcs(), max(nCells), max(nFaces), max(nPoints) ))))+2 nameLen=max(len("Points"),boundMaxLen) tab=ReST.table() tab[0]=["CPU"]+list(range(sol.nrProcs())) tab.addLine() tab[1]=["Points"]+nPoints tab[2]=["Faces"]+nFaces tab[3]=["Cells"]+nCells tab.addLine(head=True) nr=3 for b in boundaryNames: nr+=1 tab[(nr,0)]=b for i,p in enumerate(sol.processorDirs()): try: nFaces= ParsedBoundaryDict(sol.boundaryDict(processor=p, region=theRegion, time=self.opts.time), treatBinaryAsASCII=self.opts.treatBinaryAsASCII )[b]["nFaces"] except IOError: nFaces= ParsedBoundaryDict(sol.boundaryDict(processor=p, region=theRegion), treatBinaryAsASCII=self.opts.treatBinaryAsASCII )[b]["nFaces"] except KeyError: nFaces=0 tab[(nr,i+1)]=nFaces print_(tab) if self.opts.longBCreport: print_(ReST.heading("The boundary conditions for t =",time)) for b in boundaryNames: print_(ReST.buildHeading("Boundary: ",b,level=self.opts.headingLevel+1)) bound=boundary[b] print_(":Type:\t",bound["type"]) if "physicalType" in bound: print_(":Physical:\t",bound["physicalType"]) print_(":Faces:\t",bound["nFaces"]) print_() heads=["Field","type"] tab=ReST.table() tab[0]=heads tab.addLine(head=True) for row,fName in enumerate(fieldNames): tab[(row+1,0)]=fName f=fields[fName] if "boundaryField" not in f: tab[(row+1,1)]="Not a field file" elif b not in f["boundaryField"]: tab[(row+1,1)]="MISSING !!!" else: bf=f["boundaryField"][b] for k in bf: try: col=heads.index(k) except ValueError: col=len(heads) tab[(0,col)]=k heads.append(k) cont=str(bf[k]) if type(bf[k])==Field: if bf[k].isBinary(): cont= bf[k].binaryString() if cont.find("\n")>=0: tab[(row+1,col)]=cont[:cont.find("\n")]+"..." else: tab[(row+1,col)]=cont print_(tab) if self.opts.shortBCreport: print_(ReST.heading("Table of boundary conditions for t =",time)) types={} hasPhysical=False for b in boundary: if "physicalType" in boundary[b]: hasPhysical=True types[b]={} for fName in fields: f=fields[fName] try: if b not in f["boundaryField"]: types[b][fName]="MISSING" else: types[b][fName]=f["boundaryField"][b]["type"] except KeyError: types[b][fName]="Not a field" tab=ReST.table() tab[0]=[""]+boundaryNames tab.addLine() tab[(1,0)]="Patch Type" for i,b in enumerate(boundaryNames): tab[(1,i+1)]=boundary[b]["type"] nr=2 if hasPhysical: tab[(nr,0)]="Physical Type" for i,b in enumerate(boundaryNames): if "physicalType" in boundary[b]: tab[(nr,i+1)]=boundary[b]["physicalType"] nr+=1 tab[(nr,0)]="Length" for i,b in enumerate(boundaryNames): tab[(nr,i+1)]=boundary[b]["nFaces"] nr+=1 tab.addLine(head=True) for fName in fieldNames: tab[(nr,0)]=fName for i,b in enumerate(boundaryNames): tab[(nr,i+1)]=types[b][fName] nr+=1 print_(tab) if self.opts.dimensions: print_(ReST.heading("Dimensions of fields for t =",time)) tab=ReST.table() tab[0]=["Name"]+"[ kg m s K mol A cd ]".split()[1:-1] tab.addLine(head=True) for i,fName in enumerate(fieldNames): f=fields[fName] try: dim=str(f["dimensions"]).split()[1:-1] except KeyError: dim=["-"]*7 tab[i+1]=[fName]+dim print_(tab) if self.opts.internal: print_(ReST.heading("Internal value of fields for t =",time)) tab=ReST.table() tab[0]=["Name","Value"] tab.addLine(head=True) for i,fName in enumerate(fieldNames): f=fields[fName] try: if f["internalField"].isBinary(): val=f["internalField"].binaryString() else: cont=str(f["internalField"]) if cont.find("\n")>=0: val=cont[:cont.find("\n")]+"..." else: val=cont except KeyError: val="Not a field file" tab[i+1]=[fName,val] print_(tab) if self.opts.processorMatrix: print_(ReST.heading("Processor matrix")) if sol.nrProcs()<2: print_("This case is not decomposed") else: matrix=[ [0,]*sol.nrProcs() for i in range(sol.nrProcs())] for i,p in enumerate(sol.processorDirs()): try: bound=ParsedBoundaryDict(sol.boundaryDict(processor=p, region=theRegion, time=self.opts.time) ,treatBinaryAsASCII=self.opts.treatBinaryAsASCII) except IOError: bound=ParsedBoundaryDict(sol.boundaryDict(processor=p, treatBinaryAsASCII=self.opts.treatBinaryAsASCII, region=theRegion) ,treatBinaryAsASCII=self.opts.treatBinaryAsASCII) for j in range(sol.nrProcs()): name="procBoundary%dto%d" %(j,i) name2="procBoundary%dto%d" %(i,j) if name in bound: matrix[i][j]=bound[name]["nFaces"] if name2 in bound: matrix[i][j]=bound[name2]["nFaces"] print_("Matrix of processor interactions (faces)") print_() tab=ReST.table() tab[0]=["CPU"]+list(range(sol.nrProcs())) tab.addLine(head=True) for i,col in enumerate(matrix): tab[i+1]=[i]+matrix[i] print_(tab) if self.opts.linearSolvers: print_(ReST.heading("Linear Solvers")) linTable=ReST.table() fvSol=ParsedParameterFile(path.join(sol.systemDir(),"fvSolution"), treatBinaryAsASCII=self.opts.treatBinaryAsASCII) allInfo={} for sName in fvSol["solvers"]: raw=fvSol["solvers"][sName] info={} if type(raw) in [dict,DictProxy]: # fvSolution format in 1.7 info["solver"]=raw["solver"] solverData=raw else: info["solver"]=raw[0] solverData=raw[1] if type(solverData) in [dict,DictProxy]: try: info["tolerance"]=solverData["tolerance"] except KeyError: info["tolerance"]=1. try: info["relTol"]=solverData["relTol"] except KeyError: info["relTol"]=0. else: # the old (pre-1.5) fvSolution-format info["tolerance"]=solverData info["relTol"]=raw[2] allInfo[sName]=info linTable[0]=["Name","Solver","Abs. Tolerance","Relative Tol."] linTable.addLine(head=True) nr=0 for n,i in iteritems(allInfo): nr+=1 linTable[nr]=(n,i["solver"],i["tolerance"],i["relTol"]) print_(linTable) if self.opts.relaxationFactors: print_(ReST.heading("Relaxation")) fvSol=ParsedParameterFile(path.join(sol.systemDir(),"fvSolution"), treatBinaryAsASCII=self.opts.treatBinaryAsASCII) if "relaxationFactors" in fvSol: tab=ReST.table() tab[0]=["Name","Factor"] tab.addLine(head=True) nr=0 for n,f in iteritems(fvSol["relaxationFactors"]): nr+=1 tab[nr]=[n,f] print_(tab) else: print_("No relaxation factors defined for this case")
def run(self): dirs = self.parser.getArgs() if len(dirs) == 0: dirs = [path.curdir] cData = [] totalDiskusage = 0 useSolverInData = False self.hasState = False customData = [] for i, c in enumerate(self.opts.customData): lst = c.split("=") if len(lst) == 2: name, spec = lst name += "_" # Make sure that there is no collision with standard-names elif len(lst) == 1: name, spec = "Custom%d" % (i + 1), c else: self.error("Custom specification", c, "does not fit the pattern 'name=subs1::subs2::..'") customData.append((name, spec.split("::"))) if len(customData) > 0 and not self.opts.solverNameForCustom: self.warning( "Parameter '--solver-name-for-custom-data' should be set if '--custom-data' is used" ) useSolverInData = True elif self.opts.hostname: useSolverInData = True for d in tqdm(dirs, unit="dirs", disable=not self.opts.progressBar or len(dirs) < 2): if not path.isdir(d): self.warning("There is no directory", d, "here") continue for n in tqdm(listdir(d), unit="entries", desc=path.basename(path.abspath(d)), disable=not self.opts.progressBar): if not self.fnmatch(n): continue cName = path.join(d, n) if path.isdir(cName): try: sol = SolutionDirectory(cName, archive=None, paraviewLink=False) if sol.isValid(): if self.opts.progress: print_("Processing", cName) data = {} data["mtime"] = stat(cName)[ST_MTIME] times = sol.getTimes() try: data["first"] = times[0] except IndexError: data["first"] = "None" try: data["last"] = times[-1] except IndexError: data["last"] = "None" data["nrSteps"] = len(times) data["procs"] = sol.nrProcs() data["pFirst"] = -1 data["pLast"] = -1 data["nrParallel"] = -1 if self.opts.parallel: pTimes = sol.getParallelTimes() data["nrParallel"] = len(pTimes) if len(pTimes) > 0: data["pFirst"] = pTimes[0] data["pLast"] = pTimes[-1] data["name"] = cName data["diskusage"] = -1 if self.opts.diskusage: data["diskusage"] = diskUsage(cName) totalDiskusage += data["diskusage"] if self.opts.parallel: for f in listdir(cName): if re.compile("processor[0-9]+").match(f): data["mtime"] = max( stat(path.join(cName, f))[ST_MTIME], data["mtime"]) if self.opts.state: try: data["nowTime"] = float( self.readState(sol, "CurrentTime")) except ValueError: data["nowTime"] = None try: data["lastOutput"] = time.mktime( time.strptime( self.readState( sol, "LastOutputSeen"))) except ValueError: data["lastOutput"] = "nix" data["state"] = self.readState(sol, "TheState") if self.opts.state or self.opts.estimateEndTime: try: data["startedAt"] = time.mktime( time.strptime( self.readState(sol, "StartedAt"))) except ValueError: data["startedAt"] = "nix" if self.opts.startEndTime or self.opts.estimateEndTime: try: ctrlDict = ParsedParameterFile( sol.controlDict(), doMacroExpansion=True) except PyFoamParserError: # Didn't work with Macro expansion. Let's try without try: ctrlDict = ParsedParameterFile( sol.controlDict()) except PyFoamParserError: ctrlDict = None if ctrlDict: data["startTime"] = ctrlDict["startTime"] data["endTime"] = ctrlDict["endTime"] else: data["startTime"] = None data["endTime"] = None if self.opts.estimateEndTime: data["endTimeEstimate"] = None if self.readState(sol, "TheState") == "Running": gone = time.time() - data["startedAt"] try: current = float( self.readState(sol, "CurrentTime")) frac = (current - data["startTime"] ) / (data["endTime"] - data["startTime"]) except ValueError: frac = 0 if frac > 0: data["endTimeEstimate"] = data[ "startedAt"] + gone / frac if self.opts.hgInfo: if path.isdir(path.join(cName, ".hg")): from stat import ST_ATIME prevStat = stat(cName) try: data["hgInfo"] = sub.Popen( [ "hg", "id", "-R", cName, "-b", "-n", "-i" ], stdout=sub.PIPE).communicate( )[0].strip() except OSError: data["hgInfo"] = "<hg not working>" postStat = stat(cName) if prevStat[ST_MTIME] != postStat[ST_MTIME]: # hg seems to modify the modification time of the directory. So reset it os.utime(cName, (postStat[ST_ATIME], prevStat[ST_MTIME])) else: data["hgInfo"] = "<no .hg directory>" if len(customData) > 0 or self.opts.hostname: fn = None pickleFile = None if useSolverInData: data["solver"] = "none found" # try to find the oldest pickled file for f in [ "pickledData", "pickledUnfinishedData", "pickledStartData" ]: dirAndTime = [] for g in glob( path.join(cName, "*.analyzed")): pName = path.join(g, f) base = path.basename(g) if base.find("PyFoamRunner.") == 0: solverName = base[ len("PyFoamRunner." ):-len(".analyzed")] else: solverName = None if path.exists(pName): dirAndTime.append( (path.getmtime(pName), solverName, pName)) dirAndTime.sort(key=lambda x: x[0]) if len(dirAndTime) > 0: data["solver"] = dirAndTime[-1][1] pickleFile = dirAndTime[-1][2] break solverName = data["solver"] else: solverName = self.opts.solverNameForCustom if pickleFile: fn = pickleFile else: for f in [ "pickledData", "pickledUnfinishedData", "pickledStartData" ]: fp = path.join( cName, "PyFoamRunner." + solverName + ".analyzed", f) if path.exists(fp): fn = fp break if fn: raw = pickle.Unpickler(open(fn, "rb")).load() for n, spec in customData: dt = raw for k in spec: try: dt = dt[k] except KeyError: dt = "No key '" + k + "'" break if isinstance(dt, string_types): break data[n] = dt if self.opts.hostname: try: data["hostname"] = raw[ "hostname"].split(".")[0] except KeyError: data["hostname"] = "<unspecified>" else: for n, spec in customData: data[n] = "<no file>" if self.opts.hostname: data["hostname"] = "<no file>" cData.append(data) except OSError: print_(cName, "is unreadable") if self.opts.progress: print_("Sorting data") cData.sort(key=lambda x: x[self.opts.sort], reverse=self.opts.reverse) if len(cData) == 0: print_("No cases found") return if self.opts.dump: print_(cData) return lens = {} for k in list(cData[0].keys()): lens[k] = len(k) for c in cData: for k in ["mtime", "lastOutput", "startedAt", "endTimeEstimate"]: try: if c[k] != None: if self.opts.relativeTime: c[k] = datetime.timedelta( seconds=long(time.time() - c[k])) else: c[k] = time.asctime(time.localtime(c[k])) except KeyError: pass except TypeError: c[k] = None try: c["diskusage"] = humanReadableSize(c["diskusage"]) except KeyError: pass for k, v in iteritems(c): lens[k] = max(lens[k], len(str(v))) format = "" spec = ["mtime", " | "] if self.opts.hostname: spec += ["hostname", " | "] spec += ["first", " - ", "last", " (", "nrSteps", ") "] if self.opts.parallel: spec += [ "| ", "procs", " : ", "pFirst", " - ", "pLast", " (", "nrParallel", ") | " ] if self.opts.diskusage: spec += ["diskusage", " | "] if self.hasState: spec += ["nowTime", " s ", "state", " | "] if self.opts.advancedState: spec += ["lastOutput", " | ", "startedAt", " | "] if self.opts.estimateEndTime: if not self.opts.advancedState: spec += ["startedAt", " | "] spec += ["endTimeEstimate", " | "] if self.opts.startEndTime: spec += ["startTime", " | ", "endTime", " | "] if useSolverInData: spec += ["solver", " | "] for n, s in customData: spec += [n, " | "] if self.opts.hgInfo: spec += ["hgInfo", " | "] spec += ["name"] for i, l in enumerate(spec): if not l in list(cData[0].keys()): format += l else: if i < len(spec) - 1: format += "%%(%s)%ds" % (l, lens[l]) else: format += "%%(%s)s" % (l) if self.opts.progress: print_("Printing\n\n") header = format % dict( list(zip(list(cData[0].keys()), list(cData[0].keys())))) print_(header) print_("-" * len(header)) for d in cData: for k in list(d.keys()): d[k] = str(d[k]) print_(format % d) if self.opts.diskusage: print_("Total disk-usage:", humanReadableSize(totalDiskusage))
def run(self): dirs=self.parser.getArgs() if len(dirs)==0: dirs=[path.curdir] cData=[] totalDiskusage=0 self.hasState=False for d in dirs: for n in listdir(d): cName=path.join(d,n) if path.isdir(cName): try: sol=SolutionDirectory(cName,archive=None,paraviewLink=False) if sol.isValid(): if self.opts.progress: print "Processing",cName data={} data["mtime"]=stat(cName)[ST_MTIME] times=sol.getTimes() try: data["first"]=times[0] except IndexError: data["first"]="None" try: data["last"]=times[-1] except IndexError: data["last"]="None" data["nrSteps"]=len(times) data["procs"]=sol.nrProcs() data["pFirst"]=-1 data["pLast"]=-1 data["nrParallel"]=-1 if self.opts.parallel: pTimes=sol.getParallelTimes() data["nrParallel"]=len(pTimes) if len(pTimes)>0: data["pFirst"]=pTimes[0] data["pLast"]=pTimes[-1] data["name"]=cName data["diskusage"]=-1 if self.opts.diskusage: data["diskusage"]=int(subprocess.Popen(["du","-sm",cName], stdout=subprocess.PIPE).communicate()[0].split()[0]) totalDiskusage+=data["diskusage"] if self.opts.parallel: for f in listdir(cName): if re.compile("processor[0-9]+").match(f): data["mtime"]=max(stat(path.join(cName,f))[ST_MTIME],data["mtime"]) if self.opts.state: try: data["nowTime"]=float(self.readState(sol,"CurrentTime")) except ValueError: data["nowTime"]=None try: data["lastOutput"]=time.mktime(time.strptime(self.readState(sol,"LastOutputSeen"))) except ValueError: data["lastOutput"]="nix" try: data["startedAt"]=time.mktime(time.strptime(self.readState(sol,"StartedAt"))) except ValueError: data["startedAt"]="nix" data["state"]=self.readState(sol,"TheState") cData.append(data) except OSError: print cName,"is unreadable" if self.opts.progress: print "Sorting data" if self.opts.reverse: cData.sort(lambda x,y:cmp(y[self.opts.sort],x[self.opts.sort])) else: cData.sort(lambda x,y:cmp(x[self.opts.sort],y[self.opts.sort])) if len(cData)==0: print "No cases found" return if self.opts.dump: print cData return lens={} for k in cData[0].keys(): lens[k]=len(k) for c in cData: for k in ["mtime","lastOutput","startedAt"]: try: if self.opts.relativeTime: c[k]=datetime.timedelta(seconds=long(time.time()-c[k])) else: c[k]=time.asctime(time.localtime(c[k])) except KeyError: pass except TypeError: c[k]=None for k,v in c.iteritems(): lens[k]=max(lens[k],len(str(v))) format="" spec=["mtime"," | ","first"," - ","last"," (","nrSteps",") "] if self.opts.parallel: spec+=["| ","procs"," : ","pFirst"," - ","pLast"," (","nrParallel",") | "] if self.opts.diskusage: spec+=["diskusage"," MB "] if self.hasState: spec+=["nowTime"," s ","state"," | "] if self.opts.advancedState: spec+=["lastOutput"," | ","startedAt"," | "] spec+=["name"] for i,l in enumerate(spec): if not l in cData[0].keys(): format+=l else: if i<len(spec)-1: format+="%%(%s)%ds" % (l,lens[l]) else: format+="%%(%s)s" % (l) if self.opts.progress: print "Printing\n\n" header=format % dict(zip(cData[0].keys(),cData[0].keys())) print header print "-"*len(header) for d in cData: for k in d.keys(): d[k]=str(d[k]) print format % d if self.opts.diskusage: print "Total disk-usage:",totalDiskusage,"MB"