def picklePlots(self,wait=False):
"""Writes the necessary information for the plots permanently to disc,
so that it doesn't have to be generated again
:param wait: wait for the lock to be allowed to pickle"""
# print "Putting some pickles in the jar"
lines=allLines()
plots=allPlots()
if lines and plots:
gotIt=self.pickleLock.acquire(wait)
if not gotIt:
return
pickleFile=path.join(self.logDir,"pickledPlots")
pick=pickle.Pickler(open(pickleFile+".tmp","wb"))
pick.dump(lines.prepareForTransfer())
pick.dump(plots.prepareForTransfer())
move(pickleFile+".tmp",pickleFile)
if hasattr(self,"data"):
pickleFile=path.join(self.logDir,"pickledUnfinishedData")
pick=pickle.Pickler(open(pickleFile+".tmp","wb"))
pick.dump(self.data)
del pick
move(pickleFile+".tmp",pickleFile)
self.pickleLock.release()
def picklePlots(self, wait=False):
"""Writes the necessary information for the plots permanently to disc,
so that it doesn't have to be generated again
@param wait: wait for the lock to be allowed to pickle"""
# print "Putting some pickles in the jar"
lines = allLines()
plots = allPlots()
if lines and plots:
gotIt = self.pickleLock.acquire(wait)
if not gotIt:
return
pickleFile = path.join(self.logDir, "pickledPlots")
pick = pickle.Pickler(open(pickleFile + ".tmp", "wb"))
pick.dump(lines.prepareForTransfer())
pick.dump(plots.prepareForTransfer())
move(pickleFile + ".tmp", pickleFile)
if hasattr(self, "data"):
pickleFile = path.join(self.logDir, "pickledUnfinishedData")
pick = pickle.Pickler(open(pickleFile + ".tmp", "wb"))
pick.dump(self.data)
del pick
move(pickleFile + ".tmp", pickleFile)
self.pickleLock.release()
def run(self):
origPath=self.parser.getArgs()[0]
parameterFile=self.parser.getArgs()[1]
self.addLocalConfig(origPath)
self.checkCase(origPath)
nrModes=(1 if self.opts.inplaceExecution else 0) + \
(1 if self.opts.oneClonedCase else 0) + \
(1 if self.opts.listVariations else 0) + \
(1 if self.opts.everyVariantOneCase else 0)
if nrModes==0:
self.error("Specify one of the modes --list-variations, --inplace-execution, --one-cloned-case-execution or --every-variant-one-case-execution")
elif nrModes>1:
self.error("The modes --list-variations, --inplace-execution, --one-cloned-case-execution or --every-variant-one-case-execution are mutual exclusive")
if self.opts.noExecuteSolver:
if not self.opts.everyVariantOneCase and self.opts.singleVariation==None and not self.opts.listVariations:
self.error("--no-execute-solver only works with --every-variant-one-case-execution")
if not self.opts.clonedCasePrefix:
self.opts.clonedCasePrefix=path.basename(parameterFile)
if not self.opts.cloneToDirectory:
self.opts.cloneToDirectory=path.dirname(path.abspath(origPath))
if not self.opts.database:
self.opts.database=parameterFile+".database"
variationData=ParsedParameterFile(parameterFile,
noHeader=True,
noVectorOrTensor=True)
if not "values" in variationData:
self.error("Entry 'values' (dictionary) needed in",parameterFile)
if not "solver" in variationData["values"]:
self.error("Entry 'solver' (list or string) needed in 'values' in",parameterFile)
fixed={}
varied=[]
nrVariations=1
for k in variationData["values"]:
v=variationData["values"][k]
if type(v)!=list:
self.error("Entry",k,"is not a list")
if len(v)==1:
fixed[k]=v[0]
elif len(v)>1:
varied.append((k,v))
nrVariations*=len(v)
else:
self.warning("Entry",k,"is empty")
if len(varied)==0:
self.error("No parameters to vary")
self.printPhase(nrVariations,"variations with",len(varied),"parameters")
def makeVariations(vList):
name,vals=vList[0]
if len(vList)>1:
var=makeVariations(vList[1:])
variation=[]
for orig in var:
for v in vals:
d=orig.copy()
d[name]=v
variation.append(d)
return variation
else:
return [{name:v} for v in vals]
variations=makeVariations(varied)
self["variations"]=variations
self["fixed"]=fixed
if self.opts.startVariation!=None:
start=self.opts.startVariation
else:
start=0
if self.opts.endVariation!=None:
end=self.opts.endVariation
if end>=len(variations):
end=len(variations)-1
else:
end=len(variations)-1
if self.opts.singleVariation!=None:
if self.opts.startVariation or self.opts.endVariation:
self.error("--single-variation not possible with --end-variation-number or --start-variation-number")
if self.opts.singleVariation<0:
self.error("--single-variation must be greater or equal to 0")
if self.opts.singleVariation>=len(variations):
self.error("Only",len(variations))
start=self.opts.singleVariation
end =self.opts.singleVariation
if end<start:
self.error("Start value",start,"bigger than end value",end)
if self.opts.listVariations:
self.printPhase("Listing variations")
for i in range(start,end+1):
print_("Variation",i,":",variations[i])
return
if not hasDatabase or self.opts.noDatabaseWrite:
if path.exists(self.opts.database) and self.opts.createDatabase:
self.error("database-file",self.opts.database,"exists already.")
elif not path.exists(self.opts.database) and not self.opts.createDatabase and not self.opts.autoCreateDatabase:
self.error("database-file",self.opts.database,"does not exist")
createDatabase=self.opts.createDatabase
if self.opts.autoCreateDatabase and not path.exists(self.opts.database):
createDatabase=True
if not hasDatabase or self.opts.noDatabaseWrite:
db=None
else:
db=RunDatabase(self.opts.database,
create=createDatabase,
verbose=self.opts.verbose)
origCase=SolutionDirectory(origPath,archive=None)
if self.opts.oneClonedCase:
self.printPhase("Cloning work case")
workCase=origCase.cloneCase(path.join(self.opts.cloneToDirectory,
self.opts.clonedCasePrefix+"_"+path.basename(origPath)))
self.printPhase("Starting actual variations")
for i in range(start,end+1):
self.printPhase("Variation",i,"of [",start,",",end,"]")
usedVals=variations[i].copy()
usedVals.update(fixed)
self.prepareHooks()
clone=False
if self.opts.inplaceExecution:
workCase=origCase
elif self.opts.oneClonedCase:
pass
else:
self.printPhase("Cloning work case")
workCase=origCase.cloneCase(path.join(self.opts.cloneToDirectory,
self.opts.clonedCasePrefix+"_"+
("%05d" % i)+"_"+path.basename(origPath)))
self.processPlotLineOptions(autoPath=workCase.name)
self.printPhase("Setting up the case")
self.prepare(workCase,overrideParameters=usedVals)
if self.opts.noExecuteSolver:
self.printPhase("Not executing the solver")
continue
if self.opts.oneClonedCase or self.opts.inplaceExecution:
self.setLogname(self.opts.clonedCasePrefix+("_%05d_"%i)+usedVals["solver"],
useApplication=False,
force=True)
else:
self.setLogname(self.opts.clonedCasePrefix+"_"+usedVals["solver"],
useApplication=False,
force=True)
lam=self.getParallel(workCase)
allLines().clear()
allPlots().clear()
resetCustomCounter()
run=AnalyzedRunner(BoundingLogAnalyzer(progress=self.opts.progress,
doFiles=self.opts.writeFiles,
singleFile=self.opts.singleDataFilesOnly,
doTimelines=True),
silent=self.opts.progress or self.opts.silent,
argv=[usedVals["solver"],"-case",workCase.name],
server=self.opts.server,
lam=lam,
logname=self.opts.logname,
compressLog=self.opts.compress,
logTail=self.opts.logTail,
noLog=self.opts.noLog,
remark=self.opts.remark,
parameters=usedVals,
echoCommandLine=self.opts.echoCommandPrefix,
jobId=self.opts.jobId)
run.createPlots(customRegexp=self.lines_,
writeFiles=self.opts.writeFiles)
self.runPreHooks()
self.printPhase("Running")
run.start()
self.printPhase("Getting data")
self["run%05d" % i]=run.data
if db:
db.add(run.data)
self.runPostHooks()
self.reportUsage(run)
self.reportRunnerData(run)
self.printPhase("Ending variation")
def run(self):
origPath = self.parser.getArgs()[0]
parameterFile = self.parser.getArgs()[1]
self.addLocalConfig(origPath)
self.checkCase(origPath)
nrModes=(1 if self.opts.inplaceExecution else 0) + \
(1 if self.opts.oneClonedCase else 0) + \
(1 if self.opts.listVariations else 0) + \
(1 if self.opts.everyVariantOneCase else 0)
if nrModes == 0:
self.error(
"Specify one of the modes --list-variations, --inplace-execution, --one-cloned-case-execution or --every-variant-one-case-execution"
)
elif nrModes > 1:
self.error(
"The modes --list-variations, --inplace-execution, --one-cloned-case-execution or --every-variant-one-case-execution are mutual exclusive"
)
if self.opts.noExecuteSolver:
if not self.opts.everyVariantOneCase and self.opts.singleVariation == None and not self.opts.listVariations:
self.error(
"--no-execute-solver only works with --every-variant-one-case-execution"
)
if not self.opts.clonedCasePrefix:
self.opts.clonedCasePrefix = path.basename(parameterFile)
if not self.opts.clonedCasePostfix:
self.opts.clonedCasePostfix = ""
else:
self.opts.clonedCasePostfix = "." + self.opts.clonedCasePostfix
if not self.opts.cloneToDirectory:
self.opts.cloneToDirectory = path.dirname(path.abspath(origPath))
if not self.opts.database:
self.opts.database = parameterFile + ".database"
variationData = ParsedParameterFile(
parameterFile, noHeader=True,
noVectorOrTensor=True).getValueDict()
if not "values" in variationData:
self.error("Entry 'values' (dictionary) needed in", parameterFile)
if not "solver" in variationData["values"]:
self.error("Entry 'solver' (list or string) needed in 'values' in",
parameterFile)
fixed = {}
defaults = {}
varied = []
nrVariations = 1
for k in variationData["values"]:
v = variationData["values"][k]
if type(v) != list:
self.error("Entry", k, "is not a list")
if len(v) == 1:
fixed[k] = v[0]
elif len(v) > 1:
varied.append((k, v))
nrVariations *= len(v)
else:
self.warning("Entry", k, "is empty")
if "defaults" in variationData:
defaults = variationData["defaults"]
if len(varied) == 0:
self.error("No parameters to vary")
self.printPhase(nrVariations, "variations with", len(varied),
"parameters")
def makeVariations(vList):
name, vals = vList[0]
if len(vList) > 1:
var = makeVariations(vList[1:])
variation = []
for orig in var:
for v in vals:
d = orig.copy()
if isinstance(v, (dict, DictProxy)):
d.update(v)
else:
d[name] = v
variation.append(d)
return variation
else:
return [
v if isinstance(v, (dict, DictProxy)) else {
name: v
} for v in vals
]
variations = [dict(defaults, **d) for d in makeVariations(varied)]
self["variations"] = variations
self["fixed"] = fixed
if self.opts.startVariation != None:
start = self.opts.startVariation
else:
start = 0
if self.opts.endVariation != None:
end = self.opts.endVariation
if end >= len(variations):
end = len(variations) - 1
else:
end = len(variations) - 1
if self.opts.singleVariation != None:
if self.opts.startVariation or self.opts.endVariation:
self.error(
"--single-variation not possible with --end-variation-number or --start-variation-number"
)
if self.opts.singleVariation < 0:
self.error("--single-variation must be greater or equal to 0")
if self.opts.singleVariation >= len(variations):
self.error("Only", len(variations))
start = self.opts.singleVariation
end = self.opts.singleVariation
if end < start:
self.error("Start value", start, "bigger than end value", end)
if self.opts.listVariations:
self.printPhase("Listing variations")
for i in range(start, end + 1):
print_("Variation", i, ":", variations[i])
return
if not hasDatabase or self.opts.noDatabaseWrite:
if path.exists(self.opts.database) and self.opts.createDatabase:
self.error("database-file", self.opts.database,
"exists already.")
elif not path.exists(
self.opts.database
) and not self.opts.createDatabase and not self.opts.autoCreateDatabase:
self.error("database-file", self.opts.database,
"does not exist")
createDatabase = self.opts.createDatabase
if self.opts.autoCreateDatabase and not path.exists(
self.opts.database):
createDatabase = True
if not hasDatabase or self.opts.noDatabaseWrite:
db = None
else:
db = RunDatabase(self.opts.database,
create=createDatabase,
verbose=self.opts.verbose)
origCase = SolutionDirectory(origPath, archive=None)
if self.opts.oneClonedCase:
self.printPhase("Cloning work case")
workCase = origCase.cloneCase(
path.join(
self.opts.cloneToDirectory, self.opts.clonedCasePrefix +
"_" + path.basename(origPath)) +
self.opts.clonedCasePostfix)
self.printPhase("Starting actual variations")
for i in range(start, end + 1):
self.printPhase("Variation", i, "of [", start, ",", end, "]")
usedVals = variations[i].copy()
usedVals.update(fixed)
self.prepareHooks()
clone = False
if self.opts.inplaceExecution:
workCase = origCase
elif self.opts.oneClonedCase:
pass
else:
self.printPhase("Cloning work case")
workCase = origCase.cloneCase(
path.join(
self.opts.cloneToDirectory,
self.opts.clonedCasePrefix + "_" +
("%05d" % i) + "_" + path.basename(origPath)) +
self.opts.clonedCasePostfix)
self.processPlotLineOptions(autoPath=workCase.name)
self.printPhase("Setting up the case")
self.prepare(workCase, overrideParameters=usedVals)
if self.opts.noExecuteSolver:
self.printPhase("Not executing the solver")
continue
if self.opts.oneClonedCase or self.opts.inplaceExecution:
self.setLogname(self.opts.clonedCasePrefix + ("_%05d_" % i) +
usedVals["solver"],
useApplication=False,
force=True)
else:
self.setLogname(self.opts.clonedCasePrefix + "_" +
usedVals["solver"],
useApplication=False,
force=True)
lam = self.getParallel(workCase)
allLines().clear()
allPlots().clear()
resetCustomCounter()
run = AnalyzedRunner(
BoundingLogAnalyzer(progress=self.opts.progress,
doFiles=self.opts.writeFiles,
singleFile=self.opts.singleDataFilesOnly,
doTimelines=True),
silent=self.opts.progress or self.opts.silent,
splitThres=self.opts.splitDataPointsThreshold
if self.opts.doSplitDataPoints else None,
argv=[usedVals["solver"], "-case", workCase.name],
server=self.opts.server,
lam=lam,
logname=self.opts.logname,
compressLog=self.opts.compress,
logTail=self.opts.logTail,
noLog=self.opts.noLog,
remark=self.opts.remark,
parameters=usedVals,
echoCommandLine=self.opts.echoCommandPrefix,
jobId=self.opts.jobId)
run.createPlots(customRegexp=self.lines_,
splitThres=self.opts.splitDataPointsThreshold
if self.opts.doSplitDataPoints else None,
writeFiles=self.opts.writeFiles)
self.runPreHooks()
self.printPhase("Running")
run.start()
self.printPhase("Getting data")
self["run%05d" % i] = run.data
if db:
db.add(run.data)
self.runPostHooks()
self.reportUsage(run)
self.reportRunnerData(run)
self.printPhase("Ending variation")
def getPlots(self):
"""Get all the information about the plots"""
return allPlots().prepareForTransfer()
def getPlots(self):
"""Get all the information about the plots"""
return allPlots().prepareForTransfer()