def readPickledData(self):
if "inputData" in self:
if self.opts.pickledFileRead:
self.error(
"--pickled-file specified, when input data was provided via the Python-API"
)
data = self["inputData"]
else:
if not self.opts.pickledFileRead:
if sys.stdin.isatty():
self.error("The option --pickled-file has to be set")
else:
self.opts.pickledFileRead = "stdin"
if self.opts.pickledFileRead == "stdin":
pick = pickle.Unpickler(sys.stdin)
else:
pick = pickle.Unpickler(open(self.opts.pickledFileRead))
data = pick.load()
del pick
if self.opts.printStdout:
try:
print_(data["stdout"])
except KeyError:
print_("<No stdout in data>")
if self.opts.printPickledData:
import pprint
printer = pprint.PrettyPrinter()
printer.pprint(data)
return data
def run(self):
dest = self.parser.getArgs()[0]
if path.exists(dest) and self.opts.create:
self.error("database-file", dest, "exists already.")
sources = self.parser.getArgs()[1:]
db = RunDatabase(dest,
create=self.opts.create,
verbose=self.opts.verbose)
for s in sources:
if self.opts.verbose:
print_("\nProcessing file", s)
try:
data = pickle.Unpickler(open(s, "rb")).load()
except (IOError, pickle.UnpicklingError):
e = sys.exc_info()[
1] # Needed because python 2.5 does not support 'as e'
if self.opts.skipMissing:
self.warning("File", s, "missing")
continue
else:
self.error("There was a problem reading file", s, ":", e)
try:
db.add(data, update_existing=self.opts.update)
except KeyError as e:
print("Data from file {} already in database: {}".format(s, e))
def readRunInfo(self):
"""read the runInfo from a file"""
pick = pickle.Unpickler(
open(path.join(self.caseDir, "runInfo.pickle"), "rb"))
self.__runInfo = pick.load()
def pickledData(self, pickleFile):
return pickle.Unpickler(open(path.join(self.path, pickleFile),
"rb")).load()
def lookForCases(d):
for n in tqdm(listdir(d),
unit="entries",
leave=False,
desc=path.basename(path.abspath(d)),
disable=not self.opts.progressBar):
if not self.fnmatch(n):
continue
cName = path.join(d, n)
if path.isdir(cName):
try:
sol = SolutionDirectory(cName,
archive=None,
paraviewLink=False)
if sol.isValid():
if self.opts.progress:
print_("Processing", cName)
data = {}
data["mtime"] = stat(cName)[ST_MTIME]
times = sol.getTimes()
try:
data["first"] = times[0]
except IndexError:
data["first"] = "None"
try:
data["last"] = times[-1]
except IndexError:
data["last"] = "None"
data["nrSteps"] = len(times)
data["procs"] = sol.nrProcs()
data["pFirst"] = -1
data["pLast"] = -1
data["nrParallel"] = -1
if self.opts.parallel:
pTimes = sol.getParallelTimes()
data["nrParallel"] = len(pTimes)
if len(pTimes) > 0:
data["pFirst"] = pTimes[0]
data["pLast"] = pTimes[-1]
data["name"] = cName
data["diskusage"] = -1
if self.opts.diskusage:
data["diskusage"] = diskUsage(cName)
totalDiskusage += data["diskusage"]
if self.opts.parallel:
for f in listdir(cName):
if re.compile("processor[0-9]+").match(f):
data["mtime"] = max(
stat(path.join(cName,
f))[ST_MTIME],
data["mtime"])
if self.opts.state or self.opts.estimateEndTime:
try:
data["startedAt"] = time.mktime(
time.strptime(
self.readState(sol, "StartedAt")))
except ValueError:
data["startedAt"] = "nix"
if self.opts.state:
try:
data["nowTime"] = float(
self.readState(sol, "CurrentTime"))
except ValueError:
data["nowTime"] = None
try:
data["lastOutput"] = time.mktime(
time.strptime(
self.readState(
sol, "LastOutputSeen")))
except ValueError:
data["lastOutput"] = "nix"
data["state"] = self.readState(sol, "TheState")
if data["state"] == "Running":
try:
gone = time.time() - data["lastOutput"]
if gone > self.opts.deadThreshold:
data[
"state"] = "Dead " + humanReadableDuration(
gone)
except KeyError:
pass
except TypeError:
pass
if self.opts.startEndTime or self.opts.estimateEndTime:
try:
ctrlDict = ParsedParameterFile(
sol.controlDict(),
doMacroExpansion=True)
except PyFoamParserError:
# Didn't work with Macro expansion. Let's try without
try:
ctrlDict = ParsedParameterFile(
sol.controlDict())
except PyFoamParserError:
ctrlDict = None
if ctrlDict:
data["startTime"] = ctrlDict["startTime"]
data["endTime"] = ctrlDict["endTime"]
else:
data["startTime"] = None
data["endTime"] = None
if self.opts.estimateEndTime:
data["endTimeEstimate"] = None
if self.readState(sol,
"TheState") == "Running":
gone = time.time() - data["startedAt"]
try:
current = float(
self.readState(sol, "CurrentTime"))
frac = (current - data["startTime"]
) / (data["endTime"] -
data["startTime"])
except ValueError:
frac = 0
if frac > 0:
data["endTimeEstimate"] = data[
"startedAt"] + gone / frac
if self.opts.hgInfo:
if path.isdir(path.join(cName, ".hg")):
from stat import ST_ATIME
prevStat = stat(cName)
try:
data["hgInfo"] = sub.Popen(
[
"hg", "id", "-R", cName, "-b",
"-n", "-i"
],
stdout=sub.PIPE).communicate(
)[0].strip()
except OSError:
data["hgInfo"] = "<hg not working>"
postStat = stat(cName)
if prevStat[ST_MTIME] != postStat[ST_MTIME]:
# hg seems to modify the modification time of the directory. So reset it
os.utime(cName, (postStat[ST_ATIME],
prevStat[ST_MTIME]))
else:
data["hgInfo"] = "<no .hg directory>"
if len(customData) > 0 or self.opts.hostname:
fn = None
pickleFile = None
if useSolverInData:
data["solver"] = "none found"
# try to find the oldest pickled file
dirAndTime = []
for f in [
"pickledData",
"pickledUnfinishedData",
"pickledStartData"
]:
for g in glob(
path.join(cName,
"*.analyzed")):
pName = path.join(g, f)
base = path.basename(g)
if base.find("PyFoamRunner.") == 0:
solverName = base[
len("PyFoamRunner."
):-len(".analyzed")]
else:
solverName = None
if path.exists(pName):
dirAndTime.append(
(path.getmtime(pName),
solverName, pName))
dirAndTime.sort(key=lambda x: x[0])
if len(dirAndTime) > 0:
data["solver"] = dirAndTime[-1][1]
pickleFile = dirAndTime[-1][2]
solverName = data["solver"]
else:
solverName = self.opts.solverNameForCustom
if pickleFile:
fn = pickleFile
else:
for f in [
"pickledData",
"pickledUnfinishedData",
"pickledStartData"
]:
fp = path.join(
cName, "PyFoamRunner." +
solverName + ".analyzed", f)
if path.exists(fp):
fn = fp
break
pickleOK = False
if fn:
try:
raw = pickle.Unpickler(open(
fn, "rb")).load()
pickleOK = True
for n, spec in customData:
dt = raw
for k in spec:
try:
dt = dt[k]
except KeyError:
dt = "No key '" + k + "'"
break
if isinstance(
dt, string_types):
break
data[n] = dt
if self.opts.hostname:
try:
data["hostname"] = raw[
"hostname"].split(".")[0]
except KeyError:
data[
"hostname"] = "<unspecified>"
except ValueError:
pass
if not pickleOK:
for n, spec in customData:
data[n] = "<no file>"
if self.opts.hostname:
data["hostname"] = "<no file>"
cData.append(data)
elif self.opts.recursive:
# print("Recurse",cName)
lookForCases(cName)
except OSError:
print_(cName, "is unreadable")
def run(self):
dirs = self.parser.getArgs()
if len(dirs) == 0:
dirs = [path.curdir]
cData = []
totalDiskusage = 0
useSolverInData = False
self.hasState = False
customData = []
for i, c in enumerate(self.opts.customData):
lst = c.split("=")
if len(lst) == 2:
name, spec = lst
name += "_" # Make sure that there is no collision with standard-names
elif len(lst) == 1:
name, spec = "Custom%d" % (i + 1), c
else:
self.error("Custom specification", c,
"does not fit the pattern 'name=subs1::subs2::..'")
customData.append((name, spec.split("::")))
if len(customData) > 0 and not self.opts.solverNameForCustom:
self.warning(
"Parameter '--solver-name-for-custom-data' should be set if '--custom-data' is used"
)
useSolverInData = True
elif self.opts.hostname:
useSolverInData = True
for d in tqdm(dirs,
unit="dirs",
disable=not self.opts.progressBar or len(dirs) < 2):
if not path.isdir(d):
self.warning("There is no directory", d, "here")
continue
for n in tqdm(listdir(d),
unit="entries",
desc=path.basename(path.abspath(d)),
disable=not self.opts.progressBar):
if not self.fnmatch(n):
continue
cName = path.join(d, n)
if path.isdir(cName):
try:
sol = SolutionDirectory(cName,
archive=None,
paraviewLink=False)
if sol.isValid():
if self.opts.progress:
print_("Processing", cName)
data = {}
data["mtime"] = stat(cName)[ST_MTIME]
times = sol.getTimes()
try:
data["first"] = times[0]
except IndexError:
data["first"] = "None"
try:
data["last"] = times[-1]
except IndexError:
data["last"] = "None"
data["nrSteps"] = len(times)
data["procs"] = sol.nrProcs()
data["pFirst"] = -1
data["pLast"] = -1
data["nrParallel"] = -1
if self.opts.parallel:
pTimes = sol.getParallelTimes()
data["nrParallel"] = len(pTimes)
if len(pTimes) > 0:
data["pFirst"] = pTimes[0]
data["pLast"] = pTimes[-1]
data["name"] = cName
data["diskusage"] = -1
if self.opts.diskusage:
data["diskusage"] = diskUsage(cName)
totalDiskusage += data["diskusage"]
if self.opts.parallel:
for f in listdir(cName):
if re.compile("processor[0-9]+").match(f):
data["mtime"] = max(
stat(path.join(cName,
f))[ST_MTIME],
data["mtime"])
if self.opts.state:
try:
data["nowTime"] = float(
self.readState(sol, "CurrentTime"))
except ValueError:
data["nowTime"] = None
try:
data["lastOutput"] = time.mktime(
time.strptime(
self.readState(
sol, "LastOutputSeen")))
except ValueError:
data["lastOutput"] = "nix"
data["state"] = self.readState(sol, "TheState")
if self.opts.state or self.opts.estimateEndTime:
try:
data["startedAt"] = time.mktime(
time.strptime(
self.readState(sol, "StartedAt")))
except ValueError:
data["startedAt"] = "nix"
if self.opts.startEndTime or self.opts.estimateEndTime:
try:
ctrlDict = ParsedParameterFile(
sol.controlDict(),
doMacroExpansion=True)
except PyFoamParserError:
# Didn't work with Macro expansion. Let's try without
try:
ctrlDict = ParsedParameterFile(
sol.controlDict())
except PyFoamParserError:
ctrlDict = None
if ctrlDict:
data["startTime"] = ctrlDict["startTime"]
data["endTime"] = ctrlDict["endTime"]
else:
data["startTime"] = None
data["endTime"] = None
if self.opts.estimateEndTime:
data["endTimeEstimate"] = None
if self.readState(sol,
"TheState") == "Running":
gone = time.time() - data["startedAt"]
try:
current = float(
self.readState(sol, "CurrentTime"))
frac = (current - data["startTime"]
) / (data["endTime"] -
data["startTime"])
except ValueError:
frac = 0
if frac > 0:
data["endTimeEstimate"] = data[
"startedAt"] + gone / frac
if self.opts.hgInfo:
if path.isdir(path.join(cName, ".hg")):
from stat import ST_ATIME
prevStat = stat(cName)
try:
data["hgInfo"] = sub.Popen(
[
"hg", "id", "-R", cName, "-b",
"-n", "-i"
],
stdout=sub.PIPE).communicate(
)[0].strip()
except OSError:
data["hgInfo"] = "<hg not working>"
postStat = stat(cName)
if prevStat[ST_MTIME] != postStat[ST_MTIME]:
# hg seems to modify the modification time of the directory. So reset it
os.utime(cName, (postStat[ST_ATIME],
prevStat[ST_MTIME]))
else:
data["hgInfo"] = "<no .hg directory>"
if len(customData) > 0 or self.opts.hostname:
fn = None
pickleFile = None
if useSolverInData:
data["solver"] = "none found"
# try to find the oldest pickled file
for f in [
"pickledData",
"pickledUnfinishedData",
"pickledStartData"
]:
dirAndTime = []
for g in glob(
path.join(cName,
"*.analyzed")):
pName = path.join(g, f)
base = path.basename(g)
if base.find("PyFoamRunner.") == 0:
solverName = base[
len("PyFoamRunner."
):-len(".analyzed")]
else:
solverName = None
if path.exists(pName):
dirAndTime.append(
(path.getmtime(pName),
solverName, pName))
dirAndTime.sort(key=lambda x: x[0])
if len(dirAndTime) > 0:
data["solver"] = dirAndTime[-1][1]
pickleFile = dirAndTime[-1][2]
break
solverName = data["solver"]
else:
solverName = self.opts.solverNameForCustom
if pickleFile:
fn = pickleFile
else:
for f in [
"pickledData",
"pickledUnfinishedData",
"pickledStartData"
]:
fp = path.join(
cName, "PyFoamRunner." +
solverName + ".analyzed", f)
if path.exists(fp):
fn = fp
break
if fn:
raw = pickle.Unpickler(open(fn,
"rb")).load()
for n, spec in customData:
dt = raw
for k in spec:
try:
dt = dt[k]
except KeyError:
dt = "No key '" + k + "'"
break
if isinstance(dt, string_types):
break
data[n] = dt
if self.opts.hostname:
try:
data["hostname"] = raw[
"hostname"].split(".")[0]
except KeyError:
data["hostname"] = "<unspecified>"
else:
for n, spec in customData:
data[n] = "<no file>"
if self.opts.hostname:
data["hostname"] = "<no file>"
cData.append(data)
except OSError:
print_(cName, "is unreadable")
if self.opts.progress:
print_("Sorting data")
cData.sort(key=lambda x: x[self.opts.sort], reverse=self.opts.reverse)
if len(cData) == 0:
print_("No cases found")
return
if self.opts.dump:
print_(cData)
return
lens = {}
for k in list(cData[0].keys()):
lens[k] = len(k)
for c in cData:
for k in ["mtime", "lastOutput", "startedAt", "endTimeEstimate"]:
try:
if c[k] != None:
if self.opts.relativeTime:
c[k] = datetime.timedelta(
seconds=long(time.time() - c[k]))
else:
c[k] = time.asctime(time.localtime(c[k]))
except KeyError:
pass
except TypeError:
c[k] = None
try:
c["diskusage"] = humanReadableSize(c["diskusage"])
except KeyError:
pass
for k, v in iteritems(c):
lens[k] = max(lens[k], len(str(v)))
format = ""
spec = ["mtime", " | "]
if self.opts.hostname:
spec += ["hostname", " | "]
spec += ["first", " - ", "last", " (", "nrSteps", ") "]
if self.opts.parallel:
spec += [
"| ", "procs", " : ", "pFirst", " - ", "pLast", " (",
"nrParallel", ") | "
]
if self.opts.diskusage:
spec += ["diskusage", " | "]
if self.hasState:
spec += ["nowTime", " s ", "state", " | "]
if self.opts.advancedState:
spec += ["lastOutput", " | ", "startedAt", " | "]
if self.opts.estimateEndTime:
if not self.opts.advancedState:
spec += ["startedAt", " | "]
spec += ["endTimeEstimate", " | "]
if self.opts.startEndTime:
spec += ["startTime", " | ", "endTime", " | "]
if useSolverInData:
spec += ["solver", " | "]
for n, s in customData:
spec += [n, " | "]
if self.opts.hgInfo:
spec += ["hgInfo", " | "]
spec += ["name"]
for i, l in enumerate(spec):
if not l in list(cData[0].keys()):
format += l
else:
if i < len(spec) - 1:
format += "%%(%s)%ds" % (l, lens[l])
else:
format += "%%(%s)s" % (l)
if self.opts.progress:
print_("Printing\n\n")
header = format % dict(
list(zip(list(cData[0].keys()), list(cData[0].keys()))))
print_(header)
print_("-" * len(header))
for d in cData:
for k in list(d.keys()):
d[k] = str(d[k])
print_(format % d)
if self.opts.diskusage:
print_("Total disk-usage:", humanReadableSize(totalDiskusage))
def run(self):
if not self.opts.server and not self.opts.pickle:
error("No mode selected")
if self.opts.server and self.opts.pickle:
error("Both modes selected")
doPandas = self.opts.pandasData or self.opts.pandasSeries
if self.opts.server:
if len(self.parser.getArgs()) != 2:
error("Need a server and a port to be specified")
host = self.parser.getArgs()[0]
port = int(self.parser.getArgs()[1])
try:
self.server = ServerProxy("http://%s:%d" % (host, port))
methods = self.server.system.listMethods()
except socket.error:
reason = sys.exc_info()[
1] # Needed because python 2.5 does not support 'as e'
self.error("Socket error while connecting:", reason)
except ProtocolError:
reason = sys.exc_info()[
1] # Needed because python 2.5 does not support 'as e'
self.error("XMLRPC-problem", reason)
plotInfo = self.executeCommand("getPlots()")
lineInfo = self.executeCommand("getPlotData()")
else:
if len(self.parser.getArgs()) != 1:
warning("Only the first parameter is used")
fName = self.parser.getArgs()[0]
unpick = pickle.Unpickler(open(fName))
lineInfo = unpick.load()
plotInfo = unpick.load()
if not self.quiet:
print_("Found", len(plotInfo), "plots and", len(lineInfo),
"data sets")
registry = TimeLinesRegistry()
for nr, line in iteritems(lineInfo):
if not self.quiet:
print_("Adding line", nr)
TimeLineCollection(preloadData=line, registry=registry)
registry.resolveSlaves()
if (self.opts.csvFiles or self.opts.excelFiles or doPandas
or self.opts.numpyData) and self.opts.rawLines:
rawData = {}
rawSeries = {}
rawNumpy = {}
for k, l in iteritems(registry.lines):
name = str(k)
if type(k) == int:
name = "Line%d" % k
csvName = self.opts.filePrefix + name + ".csv"
if self.opts.csvFiles:
if not self.quiet:
print_("Writing", k, "to", csvName)
l.getData().writeCSV(csvName)
if self.opts.excelFiles:
xlsName = self.opts.filePrefix + name + ".xls"
if not self.quiet:
print_("Writing", k, "to", xlsName)
l.getData().getData().to_excel(xlsName)
if self.opts.pandasData:
rawData[k] = l.getData().getData()
if self.opts.numpyData:
rawNumpy[k] = l.getData().data.copy()
if self.opts.pandasSeries:
rawSeries[k] = l.getData().getSeries()
if self.opts.numpyData:
self.setData({"rawNumpy": rawNumpy})
if self.opts.pandasData:
self.setData({"rawData": rawData})
if self.opts.pandasSeries:
self.setData({"rawSeries": rawSeries})
if self.opts.csvFiles or self.opts.excelFiles:
return
pRegistry = PlotLinesRegistry()
plotNumpy = {}
plotData = {}
plotSeries = {}
for i, p in iteritems(plotInfo):
theId = p["id"]
if not self.quiet:
print_("Plotting", i, ":", theId, end=" ")
spec = CustomPlotInfo(raw=p["spec"])
if len(registry.get(p["data"]).getTimes()) > 0 and registry.get(
p["data"]).getValueNames() > 0:
if self.opts.csvFiles or self.opts.excelFiles or doPandas or self.opts.numpyData:
dataSet = registry.get(p["data"]).getData()
if self.opts.csvFiles:
dataSet.writeCSV(self.opts.filePrefix + theId + ".csv")
if self.opts.excelFiles:
dataSet.getData().to_excel(self.opts.filePrefix +
theId + ".xls")
if self.opts.numpyData:
plotNumpy[theId] = dataSet.data.copy()
if self.opts.pandasData:
plotData[theId] = dataSet.getData()
if self.opts.pandasSeries:
plotSeries[theId] = dataSet.getSeries()
else:
if self.opts.start or self.opts.end:
# rewrite CustomPlotInfo one of these days
if "start" in spec.getDict():
self.warning("Overriding plot start",
spec["start"], "with",
self.opts.start)
spec.set("start", self.opts.start)
if "end" in spec.getDict():
self.warning("Overriding plot end", spec["end"],
"with", self.opts.end)
spec.set("end", self.opts.end)
mp = createPlotTimelines(
registry.get(p["data"]),
spec,
implementation=self.opts.implementation,
showWindow=self.opts.showWindow,
registry=pRegistry)
if self.opts.insertTitles:
mp.actualSetTitle(p["spec"]["theTitle"])
if self.opts.writePictures:
if mp.hasData():
mp.doHardcopy(self.opts.prefix + theId, "png")
else:
if not self.quiet:
print_("has no data", end=" ")
if not self.quiet:
print_()
else:
if not self.quiet:
print_("No data - skipping")
if not (self.opts.csvFiles or doPandas):
sleep(self.opts.sleepTime
) # there seems to be a timing issue with Gnuplot
if self.opts.numpyData:
self.setData({"plotNumpy": plotNumpy})
if self.opts.pandasData:
self.setData({"plotData": plotData})
if self.opts.pandasSeries:
self.setData({"plotSeries": plotSeries})