def picklePlots(self,wait=False):
        """Writes the necessary information for the plots permanently to disc,
        so that it doesn't have to be generated again
        :param wait: wait for the lock to be allowed to pickle"""

        #        print "Putting some pickles in the jar"

        lines=allLines()
        plots=allPlots()
        if lines and plots:
            gotIt=self.pickleLock.acquire(wait)
            if not gotIt:
                return

            pickleFile=path.join(self.logDir,"pickledPlots")
            pick=pickle.Pickler(open(pickleFile+".tmp","wb"))
            pick.dump(lines.prepareForTransfer())
            pick.dump(plots.prepareForTransfer())
            move(pickleFile+".tmp",pickleFile)

            if hasattr(self,"data"):
                pickleFile=path.join(self.logDir,"pickledUnfinishedData")
                pick=pickle.Pickler(open(pickleFile+".tmp","wb"))
                pick.dump(self.data)
                del pick
                move(pickleFile+".tmp",pickleFile)

            self.pickleLock.release()
Ejemplo n.º 2
0
    def picklePlots(self, wait=False):
        """Writes the necessary information for the plots permanently to disc,
        so that it doesn't have to be generated again
        @param wait: wait for the lock to be allowed to pickle"""

        #        print "Putting some pickles in the jar"

        lines = allLines()
        plots = allPlots()
        if lines and plots:
            gotIt = self.pickleLock.acquire(wait)
            if not gotIt:
                return

            pickleFile = path.join(self.logDir, "pickledPlots")
            pick = pickle.Pickler(open(pickleFile + ".tmp", "wb"))
            pick.dump(lines.prepareForTransfer())
            pick.dump(plots.prepareForTransfer())
            move(pickleFile + ".tmp", pickleFile)

            if hasattr(self, "data"):
                pickleFile = path.join(self.logDir, "pickledUnfinishedData")
                pick = pickle.Pickler(open(pickleFile + ".tmp", "wb"))
                pick.dump(self.data)
                del pick
                move(pickleFile + ".tmp", pickleFile)

            self.pickleLock.release()
Ejemplo n.º 3
0
    def run(self):
        origPath=self.parser.getArgs()[0]
        parameterFile=self.parser.getArgs()[1]

        self.addLocalConfig(origPath)
        self.checkCase(origPath)

        nrModes=(1 if self.opts.inplaceExecution else 0) + \
                (1 if self.opts.oneClonedCase else 0) + \
                (1 if self.opts.listVariations else 0) + \
                (1 if self.opts.everyVariantOneCase else 0)
        if nrModes==0:
            self.error("Specify one of the modes --list-variations, --inplace-execution, --one-cloned-case-execution or --every-variant-one-case-execution")
        elif nrModes>1:
            self.error("The modes --list-variations, --inplace-execution, --one-cloned-case-execution or --every-variant-one-case-execution are mutual exclusive")
        if self.opts.noExecuteSolver:
            if not self.opts.everyVariantOneCase and self.opts.singleVariation==None and not self.opts.listVariations:
                self.error("--no-execute-solver only works with --every-variant-one-case-execution")

        if not self.opts.clonedCasePrefix:
            self.opts.clonedCasePrefix=path.basename(parameterFile)
        if not self.opts.cloneToDirectory:
            self.opts.cloneToDirectory=path.dirname(path.abspath(origPath))
        if not self.opts.database:
            self.opts.database=parameterFile+".database"

        variationData=ParsedParameterFile(parameterFile,
                                          noHeader=True,
                                          noVectorOrTensor=True)
        if not "values" in variationData:
            self.error("Entry 'values' (dictionary) needed in",parameterFile)
        if not "solver" in variationData["values"]:
            self.error("Entry 'solver' (list or string) needed in 'values' in",parameterFile)

        fixed={}
        varied=[]
        nrVariations=1

        for k in variationData["values"]:
            v=variationData["values"][k]
            if type(v)!=list:
                self.error("Entry",k,"is not a list")
            if len(v)==1:
                fixed[k]=v[0]
            elif len(v)>1:
                varied.append((k,v))
                nrVariations*=len(v)
            else:
                self.warning("Entry",k,"is empty")

        if len(varied)==0:
            self.error("No parameters to vary")

        self.printPhase(nrVariations,"variations with",len(varied),"parameters")

        def makeVariations(vList):
            name,vals=vList[0]
            if len(vList)>1:
                var=makeVariations(vList[1:])
                variation=[]
                for orig in var:
                    for v in vals:
                        d=orig.copy()
                        d[name]=v
                        variation.append(d)
                return variation
            else:
                return [{name:v} for v in vals]

        variations=makeVariations(varied)
        self["variations"]=variations
        self["fixed"]=fixed

        if self.opts.startVariation!=None:
            start=self.opts.startVariation
        else:
            start=0
        if self.opts.endVariation!=None:
            end=self.opts.endVariation
            if end>=len(variations):
                end=len(variations)-1
        else:
            end=len(variations)-1

        if self.opts.singleVariation!=None:
            if self.opts.startVariation or self.opts.endVariation:
                self.error("--single-variation not possible with --end-variation-number or --start-variation-number")
            if self.opts.singleVariation<0:
                self.error("--single-variation must be greater or equal to 0")
            if self.opts.singleVariation>=len(variations):
                self.error("Only",len(variations))
            start=self.opts.singleVariation
            end  =self.opts.singleVariation

        if end<start:
            self.error("Start value",start,"bigger than end value",end)

        if self.opts.listVariations:
            self.printPhase("Listing variations")
            for i in range(start,end+1):
                print_("Variation",i,":",variations[i])
            return

        if not hasDatabase or self.opts.noDatabaseWrite:
            if path.exists(self.opts.database) and self.opts.createDatabase:
                self.error("database-file",self.opts.database,"exists already.")
            elif not path.exists(self.opts.database) and not self.opts.createDatabase and not self.opts.autoCreateDatabase:
                self.error("database-file",self.opts.database,"does not exist")

        createDatabase=self.opts.createDatabase
        if self.opts.autoCreateDatabase and not path.exists(self.opts.database):
            createDatabase=True

        if not hasDatabase or self.opts.noDatabaseWrite:
            db=None
        else:
            db=RunDatabase(self.opts.database,
                           create=createDatabase,
                           verbose=self.opts.verbose)

        origCase=SolutionDirectory(origPath,archive=None)
        if self.opts.oneClonedCase:
            self.printPhase("Cloning work case")
            workCase=origCase.cloneCase(path.join(self.opts.cloneToDirectory,
                                              self.opts.clonedCasePrefix+"_"+path.basename(origPath)))

        self.printPhase("Starting actual variations")

        for i in range(start,end+1):
            self.printPhase("Variation",i,"of [",start,",",end,"]")

            usedVals=variations[i].copy()
            usedVals.update(fixed)

            self.prepareHooks()

            clone=False
            if self.opts.inplaceExecution:
                workCase=origCase
            elif self.opts.oneClonedCase:
                pass
            else:
                self.printPhase("Cloning work case")
                workCase=origCase.cloneCase(path.join(self.opts.cloneToDirectory,
                                                  self.opts.clonedCasePrefix+"_"+
                                                  ("%05d" % i)+"_"+path.basename(origPath)))

            self.processPlotLineOptions(autoPath=workCase.name)

            self.printPhase("Setting up the case")

            self.prepare(workCase,overrideParameters=usedVals)

            if self.opts.noExecuteSolver:
                self.printPhase("Not executing the solver")
                continue

            if self.opts.oneClonedCase or self.opts.inplaceExecution:
                self.setLogname(self.opts.clonedCasePrefix+("_%05d_"%i)+usedVals["solver"],
                                useApplication=False,
                                force=True)
            else:
                self.setLogname(self.opts.clonedCasePrefix+"_"+usedVals["solver"],
                                useApplication=False,
                                force=True)

            lam=self.getParallel(workCase)

            allLines().clear()
            allPlots().clear()
            resetCustomCounter()

            run=AnalyzedRunner(BoundingLogAnalyzer(progress=self.opts.progress,
                                                   doFiles=self.opts.writeFiles,
                                                   singleFile=self.opts.singleDataFilesOnly,
                                                   doTimelines=True),
                               silent=self.opts.progress or self.opts.silent,
                               argv=[usedVals["solver"],"-case",workCase.name],
                               server=self.opts.server,
                               lam=lam,
                               logname=self.opts.logname,
                               compressLog=self.opts.compress,
                               logTail=self.opts.logTail,
                               noLog=self.opts.noLog,
                               remark=self.opts.remark,
                               parameters=usedVals,
                               echoCommandLine=self.opts.echoCommandPrefix,
                               jobId=self.opts.jobId)

            run.createPlots(customRegexp=self.lines_,
                            writeFiles=self.opts.writeFiles)

            self.runPreHooks()

            self.printPhase("Running")

            run.start()

            self.printPhase("Getting data")

            self["run%05d" % i]=run.data
            if db:
                db.add(run.data)

            self.runPostHooks()

            self.reportUsage(run)
            self.reportRunnerData(run)

        self.printPhase("Ending variation")
Ejemplo n.º 4
0
    def run(self):
        origPath = self.parser.getArgs()[0]
        parameterFile = self.parser.getArgs()[1]

        self.addLocalConfig(origPath)
        self.checkCase(origPath)

        nrModes=(1 if self.opts.inplaceExecution else 0) + \
                (1 if self.opts.oneClonedCase else 0) + \
                (1 if self.opts.listVariations else 0) + \
                (1 if self.opts.everyVariantOneCase else 0)
        if nrModes == 0:
            self.error(
                "Specify one of the modes --list-variations, --inplace-execution, --one-cloned-case-execution or --every-variant-one-case-execution"
            )
        elif nrModes > 1:
            self.error(
                "The modes --list-variations, --inplace-execution, --one-cloned-case-execution or --every-variant-one-case-execution are mutual exclusive"
            )
        if self.opts.noExecuteSolver:
            if not self.opts.everyVariantOneCase and self.opts.singleVariation == None and not self.opts.listVariations:
                self.error(
                    "--no-execute-solver only works with --every-variant-one-case-execution"
                )

        if not self.opts.clonedCasePrefix:
            self.opts.clonedCasePrefix = path.basename(parameterFile)
        if not self.opts.clonedCasePostfix:
            self.opts.clonedCasePostfix = ""
        else:
            self.opts.clonedCasePostfix = "." + self.opts.clonedCasePostfix
        if not self.opts.cloneToDirectory:
            self.opts.cloneToDirectory = path.dirname(path.abspath(origPath))
        if not self.opts.database:
            self.opts.database = parameterFile + ".database"

        variationData = ParsedParameterFile(
            parameterFile, noHeader=True,
            noVectorOrTensor=True).getValueDict()
        if not "values" in variationData:
            self.error("Entry 'values' (dictionary) needed in", parameterFile)
        if not "solver" in variationData["values"]:
            self.error("Entry 'solver' (list or string) needed in 'values' in",
                       parameterFile)

        fixed = {}
        defaults = {}
        varied = []
        nrVariations = 1

        for k in variationData["values"]:
            v = variationData["values"][k]
            if type(v) != list:
                self.error("Entry", k, "is not a list")
            if len(v) == 1:
                fixed[k] = v[0]
            elif len(v) > 1:
                varied.append((k, v))
                nrVariations *= len(v)
            else:
                self.warning("Entry", k, "is empty")

        if "defaults" in variationData:
            defaults = variationData["defaults"]

        if len(varied) == 0:
            self.error("No parameters to vary")

        self.printPhase(nrVariations, "variations with", len(varied),
                        "parameters")

        def makeVariations(vList):
            name, vals = vList[0]
            if len(vList) > 1:
                var = makeVariations(vList[1:])
                variation = []
                for orig in var:
                    for v in vals:
                        d = orig.copy()
                        if isinstance(v, (dict, DictProxy)):
                            d.update(v)
                        else:
                            d[name] = v
                        variation.append(d)
                return variation
            else:
                return [
                    v if isinstance(v, (dict, DictProxy)) else {
                        name: v
                    } for v in vals
                ]

        variations = [dict(defaults, **d) for d in makeVariations(varied)]
        self["variations"] = variations
        self["fixed"] = fixed

        if self.opts.startVariation != None:
            start = self.opts.startVariation
        else:
            start = 0
        if self.opts.endVariation != None:
            end = self.opts.endVariation
            if end >= len(variations):
                end = len(variations) - 1
        else:
            end = len(variations) - 1

        if self.opts.singleVariation != None:
            if self.opts.startVariation or self.opts.endVariation:
                self.error(
                    "--single-variation not possible with --end-variation-number or --start-variation-number"
                )
            if self.opts.singleVariation < 0:
                self.error("--single-variation must be greater or equal to 0")
            if self.opts.singleVariation >= len(variations):
                self.error("Only", len(variations))
            start = self.opts.singleVariation
            end = self.opts.singleVariation

        if end < start:
            self.error("Start value", start, "bigger than end value", end)

        if self.opts.listVariations:
            self.printPhase("Listing variations")
            for i in range(start, end + 1):
                print_("Variation", i, ":", variations[i])
            return

        if not hasDatabase or self.opts.noDatabaseWrite:
            if path.exists(self.opts.database) and self.opts.createDatabase:
                self.error("database-file", self.opts.database,
                           "exists already.")
            elif not path.exists(
                    self.opts.database
            ) and not self.opts.createDatabase and not self.opts.autoCreateDatabase:
                self.error("database-file", self.opts.database,
                           "does not exist")

        createDatabase = self.opts.createDatabase
        if self.opts.autoCreateDatabase and not path.exists(
                self.opts.database):
            createDatabase = True

        if not hasDatabase or self.opts.noDatabaseWrite:
            db = None
        else:
            db = RunDatabase(self.opts.database,
                             create=createDatabase,
                             verbose=self.opts.verbose)

        origCase = SolutionDirectory(origPath, archive=None)
        if self.opts.oneClonedCase:
            self.printPhase("Cloning work case")
            workCase = origCase.cloneCase(
                path.join(
                    self.opts.cloneToDirectory, self.opts.clonedCasePrefix +
                    "_" + path.basename(origPath)) +
                self.opts.clonedCasePostfix)

        self.printPhase("Starting actual variations")

        for i in range(start, end + 1):
            self.printPhase("Variation", i, "of [", start, ",", end, "]")

            usedVals = variations[i].copy()
            usedVals.update(fixed)

            self.prepareHooks()

            clone = False
            if self.opts.inplaceExecution:
                workCase = origCase
            elif self.opts.oneClonedCase:
                pass
            else:
                self.printPhase("Cloning work case")
                workCase = origCase.cloneCase(
                    path.join(
                        self.opts.cloneToDirectory,
                        self.opts.clonedCasePrefix + "_" +
                        ("%05d" % i) + "_" + path.basename(origPath)) +
                    self.opts.clonedCasePostfix)

            self.processPlotLineOptions(autoPath=workCase.name)

            self.printPhase("Setting up the case")

            self.prepare(workCase, overrideParameters=usedVals)

            if self.opts.noExecuteSolver:
                self.printPhase("Not executing the solver")
                continue

            if self.opts.oneClonedCase or self.opts.inplaceExecution:
                self.setLogname(self.opts.clonedCasePrefix + ("_%05d_" % i) +
                                usedVals["solver"],
                                useApplication=False,
                                force=True)
            else:
                self.setLogname(self.opts.clonedCasePrefix + "_" +
                                usedVals["solver"],
                                useApplication=False,
                                force=True)

            lam = self.getParallel(workCase)

            allLines().clear()
            allPlots().clear()
            resetCustomCounter()

            run = AnalyzedRunner(
                BoundingLogAnalyzer(progress=self.opts.progress,
                                    doFiles=self.opts.writeFiles,
                                    singleFile=self.opts.singleDataFilesOnly,
                                    doTimelines=True),
                silent=self.opts.progress or self.opts.silent,
                splitThres=self.opts.splitDataPointsThreshold
                if self.opts.doSplitDataPoints else None,
                argv=[usedVals["solver"], "-case", workCase.name],
                server=self.opts.server,
                lam=lam,
                logname=self.opts.logname,
                compressLog=self.opts.compress,
                logTail=self.opts.logTail,
                noLog=self.opts.noLog,
                remark=self.opts.remark,
                parameters=usedVals,
                echoCommandLine=self.opts.echoCommandPrefix,
                jobId=self.opts.jobId)

            run.createPlots(customRegexp=self.lines_,
                            splitThres=self.opts.splitDataPointsThreshold
                            if self.opts.doSplitDataPoints else None,
                            writeFiles=self.opts.writeFiles)

            self.runPreHooks()

            self.printPhase("Running")

            run.start()

            self.printPhase("Getting data")

            self["run%05d" % i] = run.data
            if db:
                db.add(run.data)

            self.runPostHooks()

            self.reportUsage(run)
            self.reportRunnerData(run)

        self.printPhase("Ending variation")
Ejemplo n.º 5
0
 def getPlots(self):
     """Get all the information about the plots"""
     return allPlots().prepareForTransfer()
Ejemplo n.º 6
0
 def getPlots(self):
     """Get all the information about the plots"""
     return allPlots().prepareForTransfer()